fsanyoto commited on
Commit
4adefb7
·
verified ·
1 Parent(s): bf31e36

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. RELEASES.json +7 -1
  2. VERSION +1 -1
  3. api/ai_review.py +591 -591
  4. api/automation_engine.py +0 -0
  5. api/connectors_tt.py +568 -568
  6. api/main.py +0 -0
  7. api/odoo_relational.py +0 -0
  8. api/providers.py +0 -0
  9. api/routes_admin.py +0 -0
  10. api/routes_agent_harness.py +455 -455
  11. api/routes_alerts.py +668 -668
  12. api/routes_automation.py +0 -0
  13. api/routes_customers.py +0 -0
  14. api/routes_geo.py +369 -369
  15. api/routes_grid.py +0 -0
  16. api/routes_keychain.py +0 -0
  17. api/routes_nav.py +0 -0
  18. api/routes_oauth.py +114 -114
  19. api/routes_publish.py +0 -0
  20. api/routes_records.py +211 -211
  21. api/routes_script_views.py +439 -439
  22. api/routes_shares.py +458 -452
  23. api/routes_slack.py +772 -772
  24. api/routes_statements.py +298 -298
  25. api/routes_tables.py +0 -0
  26. api/routes_web_agent.py +84 -84
  27. platform/aios_grid.py +0 -0
  28. platform/aios_grid_fields.json +581 -581
  29. platform/core/perm_scope.py +0 -0
  30. platform/core/script_sandbox.py +519 -519
  31. platform/core/shares.py +296 -296
  32. platform/core/store.py +0 -0
  33. platform/core/table_store.py +797 -797
  34. platform/core/user_tables.py +0 -0
  35. platform/core/users.py +532 -532
  36. platform/harness/datastore.py +0 -0
  37. platform/harness/meta_store.py +521 -521
  38. platform/harness/semantic.py +0 -0
  39. platform/model/metrics/sales.yml +149 -149
  40. platform/model/topics/odoo_accounts.yml +60 -60
  41. platform/model/topics/odoo_agents.yml +91 -91
  42. platform/model/topics/odoo_bills.yml +86 -86
  43. platform/model/topics/odoo_customers.yml +213 -213
  44. platform/model/topics/odoo_invoices.yml +98 -98
  45. platform/model/topics/odoo_orders.yml +87 -87
  46. platform/model/topics/odoo_products.yml +198 -198
  47. platform/model/topics/odoo_vendors.yml +103 -103
  48. platform/modules/agent.py +385 -385
  49. platform/modules/collections_send.py +267 -267
  50. platform/modules/product_data.py +0 -0
RELEASES.json CHANGED
@@ -1,6 +1,12 @@
1
  {
2
- "current": "624b9af",
3
  "releases": [
 
 
 
 
 
 
4
  {
5
  "version": "v29",
6
  "sha": "55ced50",
 
1
  {
2
+ "current": "v30 (71776ca)",
3
  "releases": [
4
+ {
5
+ "version": "v30",
6
+ "sha": "71776ca",
7
+ "date": "2026-08-20",
8
+ "subject": "v30: W39-T29 Member-owned View first-share hotfix"
9
+ },
10
  {
11
  "version": "v29",
12
  "sha": "55ced50",
VERSION CHANGED
@@ -1 +1 @@
1
- 624b9af
 
1
+ v30 (71776ca)
api/ai_review.py CHANGED
@@ -1,591 +1,591 @@
1
- """AI REVIEW — let a model decide which stage a card moves to (wave 23, owner ruling R4/R14).
2
-
3
- ⭐ WHAT THIS IS. A review stage holds a record until somebody decides where it goes next. R4 made
4
- that somebody optionally a MODEL: the engine hands over the record's own values, the review's
5
- prompt, and the list of stages the review is allowed to send a card to, and gets back ONE of
6
- those stage labels plus a one-line reason. Every decision is written to the same `reviews` audit
7
- log a human click writes to, tagged `by: "ai"` with the provider and model that made it.
8
-
9
- ⛔ FAIL-CLOSED IN EVERY DIRECTION, and this is the whole safety story. No key configured, a
10
- network failure, a slow answer, a malformed answer, or an answer naming a stage the review does
11
- not offer — all return `("", {...})`, and the caller leaves the card exactly where a human would
12
- have found it. The feature can be absent, broken or wrong and the worst outcome is a person doing
13
- the work. Nothing here can move a card somewhere the review does not already permit.
14
-
15
- ⭐ CHEAP FIRST (owner R14, verbatim: *"Claude is a bit too expensive"*). The ladder is
16
- `groq → cerebras → openrouter → anthropic`; the first CONFIGURED provider wins, and Anthropic is
17
- last rather than absent — it is the quality backstop, not the default. `AIOS_AI_REVIEW_PROVIDER`
18
- pins one; `AIOS_AI_MODEL` overrides the model.
19
-
20
- ⭐⭐ THE ANTHROPIC LEG IS ON THE OFFICIAL SDK NOW (D-346, W37-T39, 2026-08-19). ⛔ THIS PARAGRAPH
21
- USED TO ARGUE THE OPPOSITE and is rewritten rather than deleted, because a comment left asserting
22
- the reverse of its own code is [[two-gates-can-assert-opposite-things]] with no gate to catch it.
23
- The retired text said raw HTTP was *"a deliberate deviation … booked as a DEBT line so the
24
- integrator can overturn it deliberately rather than by drift"*. This is that overturn.
25
-
26
- Its two reasons were real and are both answered rather than waved away:
27
- 1. *"one SDK leg beside three hand-rolled ones is two implementations of the same call."* It is
28
- not, because the leg does not live here: `providers.anthropic_send` owns the wire, prefers
29
- the SDK, and normalises BOTH transports to one `(status, body)` pair. This file gained no
30
- Anthropic knowledge — it lost some.
31
- 2. *"adding a pinned dependency rebuilds the container, a real deploy risk."* Still true, so the
32
- import is LAZY and its absence is a FALLBACK, not a crash: with no `anthropic` package the
33
- same raw POST runs and `LAST_ANTHROPIC_TRANSPORT` reports `'http'`. The pin is owed in BOTH
34
- manifests ([[pin-deps-space-rebuilds]]) and is outside every worker fence this wave.
35
- ⚠ The other three legs stay OpenAI-chat-shaped over `requests` — they have no shared SDK, and that
36
- half of the original reasoning never expired.
37
-
38
- The Messages shape below is the current one: `x-api-key` + `anthropic-version: 2023-06-01`, and
39
- `stop_reason: "refusal"` is checked BEFORE reading `content` — a refusal answers HTTP 200 with an
40
- empty content list, so code that indexes `content[0]` unconditionally breaks on it.
41
- """
42
- from __future__ import annotations
43
-
44
- import json
45
- import os
46
- import re
47
-
48
- import requests
49
-
50
- #: The ladder. Order IS the policy (R14) — cheapest capable first, Anthropic last.
51
- PROVIDERS = [
52
- # ⛔⛔ D-345, FIXED W37-T39 (2026-08-19). This read `llama-3.3-70b-versatile` and Groq RETIRED
53
- # it: a real POST with the live key answers HTTP 404 `model_not_found`, and the id is absent
54
- # from `GET /openai/v1/models`. ⚠ THE `openai/` PREFIX IS PART OF GROQ'S ID and is NOT a typo
55
- # for the cerebras rung below, which serves the same family bare as `gpt-oss-120b`.
56
- {"name": "groq", "env": "GROQ_API_KEY", "shape": "openai",
57
- "url": "https://api.groq.com/openai/v1/chat/completions",
58
- "model": "openai/gpt-oss-120b"},
59
- {"name": "cerebras", "env": "CEREBRAS_API_KEY", "shape": "openai",
60
- "url": "https://api.cerebras.ai/v1/chat/completions",
61
- "model": "gpt-oss-120b"},
62
- {"name": "openrouter", "env": "OPENROUTER_API_KEY", "shape": "openai",
63
- "url": "https://openrouter.ai/api/v1/chat/completions",
64
- "model": "openai/gpt-4o-mini"},
65
- # ⚠ haiku-class DELIBERATELY, not the default Opus tier: R14 put Anthropic on this ladder as
66
- # the backstop for a one-line classification, and this is the cheapest current Claude that
67
- # does it well. A bigger model here would be spending the owner's money to pick between two
68
- # labels it already has in front of it.
69
- {"name": "anthropic", "env": "ANTHROPIC_API_KEY", "shape": "anthropic",
70
- "url": "https://api.anthropic.com/v1/messages",
71
- # ⚠ AN ENV LEVER, NOT A NEW DEFAULT (W36-T35): haiku stays the choice for the reasons above,
72
- # and a deployment whose side rungs are out of credit can raise the tier without a release.
73
- "model": os.environ.get("AIOS_AI_REVIEW_ANTHROPIC_MODEL") or "claude-haiku-4-5"},
74
- ]
75
- ANTHROPIC_VERSION = "2023-06-01"
76
- TIMEOUT_SECONDS = float(os.environ.get("AIOS_AI_REVIEW_TIMEOUT") or 20)
77
- MAX_FIELD_CHARS = 200 # per value handed to the model
78
- MAX_FIELDS = 30 # columns handed to the model
79
- MAX_REASON = 200
80
-
81
-
82
- def ladder():
83
- """The providers that are actually usable here, in order. Empty = the feature is off."""
84
- pin = (os.environ.get("AIOS_AI_REVIEW_PROVIDER") or "").strip().lower()
85
- live = [p for p in PROVIDERS if (os.environ.get(p["env"]) or "").strip()]
86
- if pin:
87
- live = [p for p in live if p["name"] == pin]
88
- return live
89
-
90
-
91
- def configured():
92
- return bool(ladder())
93
-
94
-
95
- def _record_text(row, fields):
96
- """The record, as the model sees it. Values are truncated and the column set is bounded —
97
- an automation table can carry a 32 KB JSON blob per row (C7) and a review decision does not
98
- need it. Machine bookkeeping columns are dropped: a stage cell naming the stage the card is
99
- sitting at would be the model reading its own question back."""
100
- keys = [k for k in (fields or list((row or {}).keys()))
101
- if not str(k).startswith("stage_")][:MAX_FIELDS]
102
- lines = []
103
- for k in keys:
104
- v = (row or {}).get(k)
105
- if v is None or str(v).strip() == "":
106
- continue
107
- lines.append(f"{k}: {str(v)[:MAX_FIELD_CHARS]}")
108
- return "\n".join(lines) or "(this record has no filled-in values)"
109
-
110
-
111
- def _instruction(prompt, options, label):
112
- return (
113
- f"You are deciding what happens to one record waiting at a review step called "
114
- f"{label!r} in a workflow.\n\n"
115
- f"The person who built this workflow told you: {prompt}\n\n"
116
- f"Choose EXACTLY ONE of these next steps, by its exact name:\n"
117
- + "\n".join(f"- {o}" for o in options)
118
- + "\n\nAnswer with one line of JSON and nothing else:\n"
119
- '{"choice": "<one name from the list above>", "reason": "<one short sentence>"}\n'
120
- "If the record does not give you enough to decide, answer "
121
- '{"choice": "", "reason": "why not"} and a person will decide instead.'
122
- )
123
-
124
-
125
- def _parse(text, options):
126
- """The model's line → `(choice, reason)`. A choice that is not one of the offered stages is
127
- DISCARDED, not fuzzy-matched: the offered list is a permission boundary, and a near-miss
128
- resolved by string distance is how a card ends up somewhere nobody authorised."""
129
- raw = str(text or "").strip()
130
- obj = None
131
- m = re.search(r"\{.*\}", raw, re.S)
132
- if m:
133
- try:
134
- obj = json.loads(m.group(0))
135
- except (ValueError, TypeError):
136
- obj = None
137
- if not isinstance(obj, dict):
138
- return "", ""
139
- choice = str(obj.get("choice") or "").strip()
140
- reason = str(obj.get("reason") or "").strip()[:MAX_REASON]
141
- for opt in options:
142
- if choice.lower() == str(opt).lower():
143
- return str(opt), reason # the OFFERED spelling wins, never the model's
144
- return "", reason
145
-
146
-
147
- # ⭐⭐ WAVE 35 · W35-T41 / C7 — THE TWO TRANSPORTS RETURN THE RESPONSE BODY'S `usage`.
148
- #
149
- # ⛔ THIS FILE'S OWN HEADER USED TO BE THE PRODUCT'S CONFESSION THAT NOTHING COUNTED: `ai_enrich`'s
150
- # docstring says *"`ai_review.decide` -- the product's only other LLM entry -- has no token
151
- # accounting of ANY kind"*. R9 asks for ONE meter for every AI surface, so the counting has to reach
152
- # this transport rather than be bolted onto its caller — the caller never sees the body.
153
- #
154
- # ⚠ A THIRD RETURN VALUE, NOT A MUTATED ARGUMENT, and both call sites are in `decide` below. The
155
- # tuple grew from `(text, err)` to `(text, err, usage)`; `usage` is the raw `usage` OBJECT (or None),
156
- # because reading it is `usage_ledger`'s job and a second reader here would be the two-normalizers
157
- # shape this repo keeps paying for.
158
- def _call_openai(p, model, system, user, timeout):
159
- r = requests.post(p["url"], timeout=timeout,
160
- headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}",
161
- "Content-Type": "application/json"},
162
- json={"model": model, "max_tokens": 300, "temperature": 0,
163
- "messages": [{"role": "system", "content": system},
164
- {"role": "user", "content": user}]})
165
- if r.status_code >= 400:
166
- # ⚠ NO BODY ON A 4xx/5xx: an error envelope carries no usage, and a provider that refused
167
- # before running the model has nothing to bill. The call is still COUNTED by `decide`.
168
- return "", f"{p['name']} answered {r.status_code}", None
169
- body = r.json()
170
- choices = body.get("choices") or []
171
- if not choices:
172
- return "", f"{p['name']} returned no choices", body
173
- return str(((choices[0] or {}).get("message") or {}).get("content") or ""), "", body
174
-
175
-
176
- #: What transport the last Anthropic call actually used: `'sdk'`, `'http'`, or `''` before any.
177
- #: ⚠ PROCESS-LOCAL AND FOR REPORTING ONLY. It exists so `GET /meta` and `verify_web_agent` can say
178
- #: WHICH path ran rather than inferring it from a requirements file nobody in this lane can edit.
179
- LAST_ANTHROPIC_TRANSPORT = ""
180
-
181
-
182
- def _call_anthropic(p, model, system, user, timeout):
183
- """One classification turn on the Messages API, through `providers.anthropic_send`.
184
-
185
- ⭐ D-346 (W37-T39): this used to hand-roll the POST. It now goes through the shared wire, which
186
- prefers the OFFICIAL `anthropic` SDK and falls back to the same raw POST when the package is
187
- absent. The shape of what comes back is unchanged, deliberately: `anthropic_send` normalises
188
- both transports to `(status, body)`, so every line below this call is untouched.
189
- """
190
- global LAST_ANTHROPIC_TRANSPORT
191
- import providers as _prov # noqa: PLC0415
192
- # ⚠ NO TOOLS ON THIS PATH. The builder omits `tool_choice` for exactly that case, so there is
193
- # nothing to strip here — the rule lives in `anthropic_request`, beside the thing it constrains.
194
- req = _prov.anthropic_request(
195
- model=model, key=os.environ[p["env"]].strip(), system=system,
196
- messages=[{"role": "user", "content": user}], tools=[], max_tokens=300)
197
- status, body, transport = _prov.anthropic_send(req, timeout=timeout)
198
- LAST_ANTHROPIC_TRANSPORT = transport
199
- if status >= 400 or status == 0:
200
- # ⭐ THE SENTENCE COMES FROM THE LADDER, NOT FROM HERE. `refusal_sentence` already names the
201
- # vendor and the action for every status R4 enumerated, so a bare "anthropic answered 402"
202
- # (which is what this line used to say, and what the owner quoted back at us) cannot recur.
203
- return "", _prov.refusal_sentence("anthropic", status, json.dumps(body)[:400]), None
204
- body = body or {}
205
- # ⛔ stop_reason FIRST. A safety refusal is a successful 200 with an EMPTY content list, so
206
- # reading content[0] before this check turns a refusal into an IndexError inside a run.
207
- # ⚠ A refusal IS billed and its body carries `usage`, so the body rides back on this branch too.
208
- if body.get("stop_reason") == "refusal":
209
- return "", "anthropic declined to answer this record", body
210
- parts = [b.get("text") or "" for b in (body.get("content") or [])
211
- if isinstance(b, dict) and b.get("type") == "text"]
212
- if not parts:
213
- return "", "anthropic returned no text", body
214
- return "".join(parts), "", body
215
-
216
-
217
- def decide(*, prompt, options, row, fields=(), label="Review", timeout=None, st=None, user=""):
218
- """Pick this record's next stage. Returns `(choice, meta)`.
219
-
220
- `choice` is "" whenever a person should decide — which is every failure mode there is.
221
- `meta` carries `provider`, `model`, `reason` on success, and `problem` on refusal to answer.
222
-
223
- ⭐ W35-T41 / C7 — `st` and `user` are the USAGE LEDGER's target. Both default to absent because
224
- this function's caller is `automation_engine.ai_decide(rt, ...)`, in another lane's fence: it
225
- HAS the runtime and does not pass it yet, so until it does, a review's tokens are counted as
226
- unattributed and REPORTED by `GET /usage` rather than dropped. See `usage_ledger`'s header for
227
- why they cannot be resolved implicitly (measured: a contextvar does not survive a FastAPI
228
- dependency).
229
- """
230
- import usage_ledger # noqa: PLC0415
231
- opts = [str(o) for o in (options or []) if str(o).strip()]
232
- if not opts:
233
- return "", {"problem": "the review offers no next stages"}
234
- if not str(prompt or "").strip():
235
- return "", {"problem": "the review has no prompt for the model to follow"}
236
- live = ladder()
237
- if not live:
238
- return "", {"problem": "no AI provider is configured on this deployment"}
239
- system = _instruction(prompt, opts, label)
240
- user = "Here is the record:\n\n" + _record_text(row, fields)
241
- tmo = float(timeout or TIMEOUT_SECONDS)
242
- override = (os.environ.get("AIOS_AI_MODEL") or "").strip()
243
- problems = []
244
- for p in live:
245
- model = override or p["model"]
246
- body = None
247
- try:
248
- text, err, body = (_call_anthropic if p["shape"] == "anthropic" else _call_openai)(
249
- p, model, system, user, tmo)
250
- except Exception as e: # noqa: BLE001
251
- text, err = "", f"{p['name']} failed: {type(e).__name__}"
252
- # ⭐⭐ C7 — THE LEDGER LINE, BEFORE ANY BRANCH BELOW READS THE ANSWER.
253
- # ⛔ IT IS WRITTEN ON EVERY OUTCOME THAT REACHED A PROVIDER, including a refusal and an
254
- # unusable answer. A meter that books only successes reports a cheap week for a run that
255
- # spent its budget being declined — and a declined call is billed. The only path that does
256
- # NOT record is a transport that never reached the vendor (`body is None`), which spent
257
- # nothing.
258
- if body is not None:
259
- ins, outs = usage_ledger.tokens_from(body)
260
- usage_ledger.record("ai_review", p["name"], model, ins, outs,
261
- total=usage_ledger.total_from(body), st=st, user=user)
262
- if err:
263
- problems.append(err)
264
- continue # ladder: a dead provider degrades to the next one
265
- choice, reason = _parse(text, opts)
266
- if not choice:
267
- # The provider ANSWERED and declined (or answered unusably). That is a decision about
268
- # this record, not a fault in the provider, so it does NOT fall through to a more
269
- # expensive one — the card goes to a human, which is what the model just asked for.
270
- return "", {"provider": p["name"], "model": model,
271
- "problem": reason or "the model did not choose one of the stages"}
272
- return choice, {"provider": p["name"], "model": model, "reason": reason}
273
- return "", {"problem": "; ".join(problems)[:300] or "no provider answered"}
274
-
275
-
276
- # ══════════════════════════════════════════════════════════════════ the FLOW WRITER
277
- # ⭐⭐ WAVE 33 · W33-T54/T55 (owner item 7, ruling R3) — A PROMPT BECOMES A DRAFT AUTOMATION.
278
- #
279
- # Owner, verbatim: *"lay out the foundation with custom tools we provide"* — and the tools are the
280
- # action catalog, which has been machine-readable since wave 23. So this writes no new vocabulary:
281
- # it hands the model the SAME `ACTION_CATALOG` the menu paints, the SAME `ACTION_REQUIRED` the
282
- # runner blocks on, and the SAME trigger keys `clean_trigger` accepts, and asks for one JSON object
283
- # in that vocabulary.
284
- #
285
- # ⛔ WHY IT LIVES IN `ai_review.py` RATHER THAN A NEW FILE. Two reasons, both about ownership. This
286
- # is the one module in the API package that already holds an LLM ladder and its credentials, so a
287
- # second one would be a second place a rotated key has to be noticed; and a new `routes_*.py` needs
288
- # a `main.py` line from another lane (contract C2) to be reachable at all, which is how wave 23
289
- # shipped three finished routers 404-dead behind green gates. The DOOR is a route on
290
- # `routes_automation.py`, which is already mounted.
291
- #
292
- # ⛔⛔ CONTRACT C7 IS THE DESIGN, NOT A CHECK AT THE END: *"the flow writer may emit only kinds
293
- # present in `ACTION_CATALOG`, and its output must pass `clean_actions` unchanged."* Both halves are
294
- # enforced mechanically rather than asked for politely — the kind list is an `enum` in the schema
295
- # the model fills, and the caller runs `clean_actions` and DIFFS the result. That diff matters
296
- # because `clean_actions` has no disclosure channel (D-75): it drops a key it does not recognise
297
- # and answers 200, so a draft accepted without the diff would show a person a flow that is not the
298
- # flow the model described, with nothing anywhere saying so.
299
- #
300
- # ⚠ `chat` IS INJECTABLE, exactly as `routes_query._call_model`'s is, and it is the reason this door
301
- # can be proven end to end with no API key and no spend. A gate that can only run where a
302
- # credential exists is a gate that never runs.
303
-
304
- #: Cerebras first, NOT the cheap-first order `ladder()` uses. Same reason `routes_query` inverts it:
305
- #: this contract includes a REFUSAL, and refusing honestly ("I cannot build that from the steps you
306
- #: have") is a model property. A cheaper rung answers an impossible request with a plausible flow,
307
- #: which is worse than no flow. Override with `AIOS_FLOW_PROVIDER`.
308
- FLOW_PROVIDER_ORDER = ("cerebras", "groq", "openrouter", "anthropic")
309
- MAX_DRAFT_ACTIONS = 12 # a draft a person reads in one screen; the engine's own cap is 25
310
- MAX_PROMPT_CHARS = 2000
311
-
312
-
313
- def flow_providers(pin=None):
314
- """The rungs usable here, in THIS module's refusal-first order. Empty = the feature is off.
315
-
316
- ⭐ `pin` IS ASK D-18 (2026-08-18): the Agent chat's model toggle must configure something, and
317
- the draft door used to read `prompt` off the body and nothing else — so the key the client sent
318
- was accepted and dropped, and the picker was a control over nothing.
319
- ⚠ AN UNKNOWN OR UNCONFIGURED PIN FALLS BACK TO THE LADDER rather than refusing. A model the
320
- ladder stopped offering must not turn every later draft into an error; the caller is told which
321
- rung actually answered, which is the honest half.
322
- """
323
- by_name = {p["name"]: p for p in PROVIDERS}
324
- live = [n for n in FLOW_PROVIDER_ORDER
325
- if n in by_name and (os.environ.get(by_name[n]["env"]) or "").strip()]
326
- wanted = str(pin or os.environ.get("AIOS_FLOW_PROVIDER") or "").strip().lower()
327
- if wanted and wanted in live:
328
- return [by_name[wanted]]
329
- return [by_name[n] for n in live]
330
-
331
-
332
- def flow_schema(kinds, trigger_keys, table_keys):
333
- """The tool schema the model fills — the catalog's OWN key lists as enums.
334
-
335
- ⛔ `kind` AND `trigger` ARE ENUMS, not free strings, and that is C7's first half enforced by the
336
- transport rather than by a check afterwards. A model asked for "any action name" invents
337
- `send_slack_message` on a deployment that has no such kind, and the failure then surfaces as a
338
- 400 from `clean_actions` carrying a sentence about a word the person never typed.
339
- ⚠ `required` IS `["kind"]` ALONE, deliberately: a REFUSAL carries no name and no actions, and a
340
- schema demanding them turns an honest refusal into a provider-side 400 that reads exactly like a
341
- transport failure (`routes_query._spec_schema` carries the same note, for the same measured
342
- reason).
343
- """
344
- return {
345
- "type": "object",
346
- "properties": {
347
- "kind": {"type": "string", "enum": ["flow", "refused"],
348
- "description": "refused = this cannot be built from the steps available"},
349
- "refusal": {"type": "string",
350
- "description": "when kind=refused: ONE plain sentence naming what is "
351
- "missing, in the words a non-technical person would use"},
352
- "name": {"type": "string", "description": "a short title for the automation"},
353
- "trigger": {"type": "string", "enum": sorted(trigger_keys),
354
- "description": "what starts this automation"},
355
- "table": {"type": "string", "enum": sorted(table_keys),
356
- "description": "the database whose records this flow walks, if any"},
357
- "actions": {
358
- "type": "array",
359
- "maxItems": MAX_DRAFT_ACTIONS,
360
- "items": {
361
- "type": "object",
362
- "properties": {
363
- "kind": {"type": "string", "enum": sorted(kinds)},
364
- "why": {"type": "string",
365
- "description": "one short sentence: why this step is here"},
366
- "config": {"type": "object", "additionalProperties": True,
367
- "description": "the step's settings, using ONLY the keys named "
368
- "for that kind in the system message"},
369
- },
370
- "required": ["kind"],
371
- },
372
- },
373
- },
374
- "required": ["kind"],
375
- }
376
-
377
-
378
- def flow_system_prompt(catalog, required, triggers, tables):
379
- """What the model is told it may build with — DERIVED, never written down twice.
380
-
381
- Every list here is the server's own: the catalog rows the menu paints, the required-key table
382
- the runner blocks on, the trigger keys `clean_trigger` accepts, and this tenant's real databases
383
- with their real columns. Nothing about the vocabulary is restated by hand, so a kind added to
384
- the catalog is offered here on the same deploy and a kind removed stops being offered.
385
- """
386
- lines = ["You build small automations for a business tool. You are given the EXACT set of "
387
- "steps this tool can perform. You may use nothing else.",
388
- "", "THE STEPS YOU MAY USE:"]
389
- # ⚠ BOTH HALVES, `key (phrase)`. The KEY is what the model must write into `config` and the
390
- # PHRASE is the only human wording of that requirement anywhere — the one the runner's own
391
- # refusal sentence is built from. Keys alone leave the model guessing what `field` means on a
392
- # `web_read` (it is a column to write into, not a form field); phrases alone leave it guessing
393
- # what to call the setting. Handing it one and hoping is how a draft comes back configured
394
- # against a key the validator drops.
395
- req = {str(k): [f"{key} ({phrase})" for phrase, key in v] for k, v in (required or {}).items()}
396
- for row in catalog or []:
397
- if not row.get("ready"):
398
- continue
399
- kind = str(row.get("kind") or "")
400
- need = req.get(kind) or []
401
- lines.append(f"- {kind}: {row.get('label')} - {row.get('detail') or ''}"
402
- + (f" REQUIRED settings: {', '.join(need)}" if need else ""))
403
- lines += ["", "WHAT CAN START AN AUTOMATION:"]
404
- for t in triggers or []:
405
- if t.get("planned") or not t.get("ready", True):
406
- continue
407
- lines.append(f"- {t.get('key')}: {t.get('label')}")
408
- lines += ["", "THE DATABASES THIS PERSON HAS:"]
409
- for t in (tables or [])[:40]:
410
- cols = ", ".join(str(f.get("key")) for f in (t.get("fields") or [])[:25])
411
- lines.append(f"- {t.get('key')} ({t.get('label')}): {cols or 'no columns yet'}")
412
- if not tables:
413
- lines.append("- (none - do not name a database)")
414
- lines += [
415
- "",
416
- "RULES:",
417
- "1. Use ONLY the step kinds listed above. If what is asked needs a step that is not there, "
418
- "answer kind=refused and say plainly which capability is missing.",
419
- "2. Fill in every REQUIRED setting you can from what the person told you. Leave one blank "
420
- "rather than inventing a web address, a CSS selector or a column name.",
421
- "3. A web address must start with http:// or https://.",
422
- "4. To use a value from the record the flow is walking, write it as {{Column name}}.",
423
- "5. Name a database only from the list above, by its key.",
424
- "6. Keep it short. Fewer steps that work beat more steps that guess.",
425
- ]
426
- return "\n".join(lines)
427
-
428
-
429
- def _flow_from_tool_call(obj):
430
- """The model's tool arguments -> `(draft, refusal)`. A shape error is a refusal, never a crash."""
431
- if not isinstance(obj, dict):
432
- return None, "the assistant's answer could not be read"
433
- if str(obj.get("kind") or "") == "refused":
434
- return None, (str(obj.get("refusal") or "").strip()
435
- or "the assistant could not build this from the steps available")
436
- acts = obj.get("actions")
437
- if not isinstance(acts, list) or not acts:
438
- return None, ("the assistant did not produce any steps - try describing what should "
439
- "happen, one action at a time")
440
- out = []
441
- for a in acts[:MAX_DRAFT_ACTIONS]:
442
- if not isinstance(a, dict) or not str(a.get("kind") or "").strip():
443
- continue
444
- cfg = a.get("config")
445
- out.append({"kind": str(a["kind"]).strip(),
446
- "config": cfg if isinstance(cfg, dict) else {},
447
- "why": str(a.get("why") or "").strip()[:200]})
448
- if not out:
449
- return None, "the assistant's steps could not be read"
450
- return {"name": str(obj.get("name") or "").strip()[:80] or "New automation",
451
- "trigger": str(obj.get("trigger") or "").strip(),
452
- "table": str(obj.get("table") or "").strip(),
453
- # ⛔⛔ THE TRUNCATION IS REPORTED, and it is a STANDING RULE that it must be (owner,
454
- # 2026-08-12, W30/R6 second sentence): *"if there is lag or it can't be done, you need
455
- # to explicitly tell me why and recommend a fix"* — **a silent truncation IS the
456
- # violation, not the limit.** `acts[:MAX_DRAFT_ACTIONS]` above dropped everything past
457
- # the twelfth and said nothing, so a model that answered with a twenty-step journey had
458
- # eight steps deleted between the answer and the screen, with no key anywhere in the
459
- # response naming them. `MAX_ACTIONS` is 20, so those steps were STORABLE — this
460
- # ceiling is the draft door's own, which makes reporting it the whole obligation.
461
- # ⚠ Found by the verifier that checked this ticket, not by a gate.
462
- "asked": len([a for a in acts if isinstance(a, dict)]),
463
- "actions": out}, ""
464
-
465
-
466
- def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeout=None,
467
- st=None, user="", model=None):
468
- """A sentence -> `(draft, refusal_sentence, provider)`. ⛔ NOTHING IS SAVED HERE.
469
-
470
- Exactly one of `draft` and `refusal_sentence` is truthy — the same contract
471
- `web_agent.run_step` keeps, so a caller has no third case to get wrong.
472
-
473
- ⭐ W35-T41 / C7 — `st`/`user` are the usage ledger's target, exactly as on `decide` above and for
474
- the same reason: both of this function's callers (`automation_engine._ai_agent_plan` and
475
- `routes_automation`'s draft door) are in lane D's fence. C7 says E adds the ledger line here and
476
- D asserts it; the two keywords are what D has to pass for the line to be attributable.
477
- """
478
- # ⚠ DECLARED AT THE TOP OF THE FUNCTION, not beside the assignment inside the provider loop.
479
- # It parses either way; a reader scanning for the declaration does not look inside a `for`.
480
- global LAST_ANTHROPIC_TRANSPORT
481
- import usage_ledger # noqa: PLC0415
482
- text = str(prompt or "").strip()[:MAX_PROMPT_CHARS]
483
- if not text:
484
- return None, "type what you want the automation to do", None
485
- kinds = sorted({str(r.get("kind")) for r in (catalog or []) if r.get("ready")})
486
- if not kinds:
487
- return None, "this deployment offers no automation steps to build with", None
488
- trigger_keys = sorted({str(t.get("key")) for t in (triggers or [])
489
- if t.get("key") and not t.get("planned")}) or ["manual"]
490
- table_keys = sorted({str(t.get("key")) for t in (tables or []) if t.get("key")}) or [""]
491
- tools = [{"type": "function", "function": {
492
- "name": "build_automation",
493
- "description": "Emit the automation, or refuse.",
494
- "parameters": flow_schema(kinds, trigger_keys, table_keys)}}]
495
- messages = [{"role": "system",
496
- "content": flow_system_prompt(catalog, required, triggers, tables)},
497
- {"role": "user", "content": text}]
498
-
499
- if chat is not None:
500
- # ⚠ THE INJECTED PATH IS THE PROVEN PATH. It runs the SAME parse and the SAME refusal
501
- # branches as a live call; only the transport is replaced.
502
- draft, refusal = _flow_from_tool_call(chat(messages, tools))
503
- return draft, refusal, "injected"
504
-
505
- provs = flow_providers(model)
506
- if not provs:
507
- # ⛔ SAY SO. An AI feature that silently does nothing is indistinguishable from one that was
508
- # never built [[flag-shipped-without-its-writer]].
509
- return None, ("the assistant is not configured on this deployment, so an automation cannot "
510
- "be drafted from a description yet"), None
511
- tmo = float(timeout or TIMEOUT_SECONDS)
512
- problems = []
513
- import providers as _prov
514
- for p in provs:
515
- # ⭐⭐ W36-T35 / R4 — THE OWNER QUOTED THIS LINE BACK AT US. It used to read
516
- # `problems.append(f"{p['name']}: tool calls are not wired for this shape")` and `continue`,
517
- # so the sentence *"anthropic: tool calls are not wired for this shape"* appeared under the
518
- # Agent module verbatim. R4: *"Anthropic becomes the tool-calling path that always works."*
519
- # It is wired now, through the ONE wire in `providers`, and it matters more than it looks:
520
- # every side rung on this account is refusing today (cerebras 402, groq 404, openrouter
521
- # 402), so without this branch the drafter cannot answer at all.
522
- # ⚠ NO `effort` ON THIS DOOR — `output_config.effort` errors on Haiku 4.5, the tier this
523
- # ladder runs. The parameter is the caller's to send, which is why the wire takes it.
524
- # ⭐ D-346 (W37-T39): BOTH shapes now yield the SAME `(status, body)` pair, so every branch
525
- # below reads one thing. The anthropic side gets there through `anthropic_send` (official
526
- # SDK when installed, the identical raw POST when not); the openai side still posts here
527
- # because there is no shared SDK across three different vendors on that wire.
528
- global LAST_ANTHROPIC_TRANSPORT
529
- try:
530
- if p["shape"] == "anthropic":
531
- req = _prov.anthropic_request(
532
- model=p["model"], key=os.environ[p["env"]].strip(), system=None,
533
- messages=messages, tools=tools, max_tokens=1500, tool_choice="required")
534
- status, body, transport = _prov.anthropic_send(req, timeout=tmo)
535
- LAST_ANTHROPIC_TRANSPORT = transport
536
- else:
537
- r = requests.post(p["url"], timeout=tmo,
538
- headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}",
539
- "Content-Type": "application/json"},
540
- json={"model": p["model"], "messages": messages, "tools": tools,
541
- "tool_choice": "required", "temperature": 0.1,
542
- "max_tokens": 1500})
543
- status = r.status_code
544
- body = r.json() if (r.content and r.status_code == 200) else {"_text": r.text}
545
- except Exception as e: # noqa: BLE001
546
- problems.append(f"{p['name']}: {type(e).__name__}")
547
- continue
548
- if status != 200:
549
- # ⛔ A SENTENCE, NOT `HTTP 402` (R4's third clause). The owner read the status codes off
550
- # this very door. `refusal_sentence` also decides whether this was a CREDIT failure, and
551
- # the memo makes the next turn SKIP the empty account instead of paying for it again.
552
- # ⚠ The body is re-serialised for the substring test because the SDK path never had a
553
- # `.text` — that is the one thing this normalisation costs, and it costs nothing else.
554
- detail = json.dumps(body)[:600] if isinstance(body, dict) else str(body)[:600]
555
- if _prov.is_credit_failure(status, detail):
556
- _prov.mark_no_credit(p["name"])
557
- problems.append(_prov.refusal_sentence(p["name"], status, detail))
558
- continue
559
- try:
560
- body = body or {}
561
- if p["shape"] == "anthropic":
562
- _text, args, _refused = _prov.anthropic_read(body)
563
- if _refused:
564
- problems.append(f"{p['name']}: {_refused}")
565
- continue
566
- else:
567
- calls = (((body.get("choices") or [{}])[0].get("message") or {})
568
- .get("tool_calls") or [])
569
- args = json.loads(calls[0]["function"]["arguments"]) if calls else None
570
- except Exception as e: # noqa: BLE001
571
- problems.append(f"{p['name']}: unreadable answer ({type(e).__name__})")
572
- continue
573
- # ⭐⭐ C7 — THE LEDGER LINE. Placed after the parse rather than before it so an UNREADABLE
574
- # answer is not double-counted by the `continue` above... ⚠ which means an unreadable answer
575
- # is NOT counted at all, and that is a deliberate, disclosed loss: a body this code cannot
576
- # parse is a body whose `usage` it also cannot trust, and `body` is out of scope in that
577
- # branch by construction. The 200-with-junk case is rare and named here rather than
578
- # silently rounded to zero.
579
- ins, outs = usage_ledger.tokens_from(body)
580
- usage_ledger.record("automation_draft", p["name"], p["model"], ins, outs,
581
- total=usage_ledger.total_from(body), st=st, user=user)
582
- draft, refusal = _flow_from_tool_call(args)
583
- if draft is None and isinstance(args, dict) and str(args.get("kind")) == "refused":
584
- # ⛔ A REFUSAL IS AN ANSWER, NOT A FAULT, so it does NOT fall through to a more
585
- # expensive rung — the model has just said this cannot be built. `decide()` takes the
586
- # same posture for the same reason.
587
- return None, refusal, p["name"]
588
- if draft is not None:
589
- return draft, "", p["name"]
590
- problems.append(f"{p['name']}: {refusal}")
591
- return None, ("; ".join(problems)[:300] or "no provider answered"), None
 
1
+ """AI REVIEW — let a model decide which stage a card moves to (wave 23, owner ruling R4/R14).
2
+
3
+ ⭐ WHAT THIS IS. A review stage holds a record until somebody decides where it goes next. R4 made
4
+ that somebody optionally a MODEL: the engine hands over the record's own values, the review's
5
+ prompt, and the list of stages the review is allowed to send a card to, and gets back ONE of
6
+ those stage labels plus a one-line reason. Every decision is written to the same `reviews` audit
7
+ log a human click writes to, tagged `by: "ai"` with the provider and model that made it.
8
+
9
+ ⛔ FAIL-CLOSED IN EVERY DIRECTION, and this is the whole safety story. No key configured, a
10
+ network failure, a slow answer, a malformed answer, or an answer naming a stage the review does
11
+ not offer — all return `("", {...})`, and the caller leaves the card exactly where a human would
12
+ have found it. The feature can be absent, broken or wrong and the worst outcome is a person doing
13
+ the work. Nothing here can move a card somewhere the review does not already permit.
14
+
15
+ ⭐ CHEAP FIRST (owner R14, verbatim: *"Claude is a bit too expensive"*). The ladder is
16
+ `groq → cerebras → openrouter → anthropic`; the first CONFIGURED provider wins, and Anthropic is
17
+ last rather than absent — it is the quality backstop, not the default. `AIOS_AI_REVIEW_PROVIDER`
18
+ pins one; `AIOS_AI_MODEL` overrides the model.
19
+
20
+ ⭐⭐ THE ANTHROPIC LEG IS ON THE OFFICIAL SDK NOW (D-346, W37-T39, 2026-08-19). ⛔ THIS PARAGRAPH
21
+ USED TO ARGUE THE OPPOSITE and is rewritten rather than deleted, because a comment left asserting
22
+ the reverse of its own code is [[two-gates-can-assert-opposite-things]] with no gate to catch it.
23
+ The retired text said raw HTTP was *"a deliberate deviation … booked as a DEBT line so the
24
+ integrator can overturn it deliberately rather than by drift"*. This is that overturn.
25
+
26
+ Its two reasons were real and are both answered rather than waved away:
27
+ 1. *"one SDK leg beside three hand-rolled ones is two implementations of the same call."* It is
28
+ not, because the leg does not live here: `providers.anthropic_send` owns the wire, prefers
29
+ the SDK, and normalises BOTH transports to one `(status, body)` pair. This file gained no
30
+ Anthropic knowledge — it lost some.
31
+ 2. *"adding a pinned dependency rebuilds the container, a real deploy risk."* Still true, so the
32
+ import is LAZY and its absence is a FALLBACK, not a crash: with no `anthropic` package the
33
+ same raw POST runs and `LAST_ANTHROPIC_TRANSPORT` reports `'http'`. The pin is owed in BOTH
34
+ manifests ([[pin-deps-space-rebuilds]]) and is outside every worker fence this wave.
35
+ ⚠ The other three legs stay OpenAI-chat-shaped over `requests` — they have no shared SDK, and that
36
+ half of the original reasoning never expired.
37
+
38
+ The Messages shape below is the current one: `x-api-key` + `anthropic-version: 2023-06-01`, and
39
+ `stop_reason: "refusal"` is checked BEFORE reading `content` — a refusal answers HTTP 200 with an
40
+ empty content list, so code that indexes `content[0]` unconditionally breaks on it.
41
+ """
42
+ from __future__ import annotations
43
+
44
+ import json
45
+ import os
46
+ import re
47
+
48
+ import requests
49
+
50
+ #: The ladder. Order IS the policy (R14) — cheapest capable first, Anthropic last.
51
+ PROVIDERS = [
52
+ # ⛔⛔ D-345, FIXED W37-T39 (2026-08-19). This read `llama-3.3-70b-versatile` and Groq RETIRED
53
+ # it: a real POST with the live key answers HTTP 404 `model_not_found`, and the id is absent
54
+ # from `GET /openai/v1/models`. ⚠ THE `openai/` PREFIX IS PART OF GROQ'S ID and is NOT a typo
55
+ # for the cerebras rung below, which serves the same family bare as `gpt-oss-120b`.
56
+ {"name": "groq", "env": "GROQ_API_KEY", "shape": "openai",
57
+ "url": "https://api.groq.com/openai/v1/chat/completions",
58
+ "model": "openai/gpt-oss-120b"},
59
+ {"name": "cerebras", "env": "CEREBRAS_API_KEY", "shape": "openai",
60
+ "url": "https://api.cerebras.ai/v1/chat/completions",
61
+ "model": "gpt-oss-120b"},
62
+ {"name": "openrouter", "env": "OPENROUTER_API_KEY", "shape": "openai",
63
+ "url": "https://openrouter.ai/api/v1/chat/completions",
64
+ "model": "openai/gpt-4o-mini"},
65
+ # ⚠ haiku-class DELIBERATELY, not the default Opus tier: R14 put Anthropic on this ladder as
66
+ # the backstop for a one-line classification, and this is the cheapest current Claude that
67
+ # does it well. A bigger model here would be spending the owner's money to pick between two
68
+ # labels it already has in front of it.
69
+ {"name": "anthropic", "env": "ANTHROPIC_API_KEY", "shape": "anthropic",
70
+ "url": "https://api.anthropic.com/v1/messages",
71
+ # ⚠ AN ENV LEVER, NOT A NEW DEFAULT (W36-T35): haiku stays the choice for the reasons above,
72
+ # and a deployment whose side rungs are out of credit can raise the tier without a release.
73
+ "model": os.environ.get("AIOS_AI_REVIEW_ANTHROPIC_MODEL") or "claude-haiku-4-5"},
74
+ ]
75
+ ANTHROPIC_VERSION = "2023-06-01"
76
+ TIMEOUT_SECONDS = float(os.environ.get("AIOS_AI_REVIEW_TIMEOUT") or 20)
77
+ MAX_FIELD_CHARS = 200 # per value handed to the model
78
+ MAX_FIELDS = 30 # columns handed to the model
79
+ MAX_REASON = 200
80
+
81
+
82
+ def ladder():
83
+ """The providers that are actually usable here, in order. Empty = the feature is off."""
84
+ pin = (os.environ.get("AIOS_AI_REVIEW_PROVIDER") or "").strip().lower()
85
+ live = [p for p in PROVIDERS if (os.environ.get(p["env"]) or "").strip()]
86
+ if pin:
87
+ live = [p for p in live if p["name"] == pin]
88
+ return live
89
+
90
+
91
+ def configured():
92
+ return bool(ladder())
93
+
94
+
95
+ def _record_text(row, fields):
96
+ """The record, as the model sees it. Values are truncated and the column set is bounded —
97
+ an automation table can carry a 32 KB JSON blob per row (C7) and a review decision does not
98
+ need it. Machine bookkeeping columns are dropped: a stage cell naming the stage the card is
99
+ sitting at would be the model reading its own question back."""
100
+ keys = [k for k in (fields or list((row or {}).keys()))
101
+ if not str(k).startswith("stage_")][:MAX_FIELDS]
102
+ lines = []
103
+ for k in keys:
104
+ v = (row or {}).get(k)
105
+ if v is None or str(v).strip() == "":
106
+ continue
107
+ lines.append(f"{k}: {str(v)[:MAX_FIELD_CHARS]}")
108
+ return "\n".join(lines) or "(this record has no filled-in values)"
109
+
110
+
111
+ def _instruction(prompt, options, label):
112
+ return (
113
+ f"You are deciding what happens to one record waiting at a review step called "
114
+ f"{label!r} in a workflow.\n\n"
115
+ f"The person who built this workflow told you: {prompt}\n\n"
116
+ f"Choose EXACTLY ONE of these next steps, by its exact name:\n"
117
+ + "\n".join(f"- {o}" for o in options)
118
+ + "\n\nAnswer with one line of JSON and nothing else:\n"
119
+ '{"choice": "<one name from the list above>", "reason": "<one short sentence>"}\n'
120
+ "If the record does not give you enough to decide, answer "
121
+ '{"choice": "", "reason": "why not"} and a person will decide instead.'
122
+ )
123
+
124
+
125
+ def _parse(text, options):
126
+ """The model's line → `(choice, reason)`. A choice that is not one of the offered stages is
127
+ DISCARDED, not fuzzy-matched: the offered list is a permission boundary, and a near-miss
128
+ resolved by string distance is how a card ends up somewhere nobody authorised."""
129
+ raw = str(text or "").strip()
130
+ obj = None
131
+ m = re.search(r"\{.*\}", raw, re.S)
132
+ if m:
133
+ try:
134
+ obj = json.loads(m.group(0))
135
+ except (ValueError, TypeError):
136
+ obj = None
137
+ if not isinstance(obj, dict):
138
+ return "", ""
139
+ choice = str(obj.get("choice") or "").strip()
140
+ reason = str(obj.get("reason") or "").strip()[:MAX_REASON]
141
+ for opt in options:
142
+ if choice.lower() == str(opt).lower():
143
+ return str(opt), reason # the OFFERED spelling wins, never the model's
144
+ return "", reason
145
+
146
+
147
+ # ⭐⭐ WAVE 35 · W35-T41 / C7 — THE TWO TRANSPORTS RETURN THE RESPONSE BODY'S `usage`.
148
+ #
149
+ # ⛔ THIS FILE'S OWN HEADER USED TO BE THE PRODUCT'S CONFESSION THAT NOTHING COUNTED: `ai_enrich`'s
150
+ # docstring says *"`ai_review.decide` -- the product's only other LLM entry -- has no token
151
+ # accounting of ANY kind"*. R9 asks for ONE meter for every AI surface, so the counting has to reach
152
+ # this transport rather than be bolted onto its caller — the caller never sees the body.
153
+ #
154
+ # ⚠ A THIRD RETURN VALUE, NOT A MUTATED ARGUMENT, and both call sites are in `decide` below. The
155
+ # tuple grew from `(text, err)` to `(text, err, usage)`; `usage` is the raw `usage` OBJECT (or None),
156
+ # because reading it is `usage_ledger`'s job and a second reader here would be the two-normalizers
157
+ # shape this repo keeps paying for.
158
+ def _call_openai(p, model, system, user, timeout):
159
+ r = requests.post(p["url"], timeout=timeout,
160
+ headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}",
161
+ "Content-Type": "application/json"},
162
+ json={"model": model, "max_tokens": 300, "temperature": 0,
163
+ "messages": [{"role": "system", "content": system},
164
+ {"role": "user", "content": user}]})
165
+ if r.status_code >= 400:
166
+ # ⚠ NO BODY ON A 4xx/5xx: an error envelope carries no usage, and a provider that refused
167
+ # before running the model has nothing to bill. The call is still COUNTED by `decide`.
168
+ return "", f"{p['name']} answered {r.status_code}", None
169
+ body = r.json()
170
+ choices = body.get("choices") or []
171
+ if not choices:
172
+ return "", f"{p['name']} returned no choices", body
173
+ return str(((choices[0] or {}).get("message") or {}).get("content") or ""), "", body
174
+
175
+
176
+ #: What transport the last Anthropic call actually used: `'sdk'`, `'http'`, or `''` before any.
177
+ #: ⚠ PROCESS-LOCAL AND FOR REPORTING ONLY. It exists so `GET /meta` and `verify_web_agent` can say
178
+ #: WHICH path ran rather than inferring it from a requirements file nobody in this lane can edit.
179
+ LAST_ANTHROPIC_TRANSPORT = ""
180
+
181
+
182
+ def _call_anthropic(p, model, system, user, timeout):
183
+ """One classification turn on the Messages API, through `providers.anthropic_send`.
184
+
185
+ ⭐ D-346 (W37-T39): this used to hand-roll the POST. It now goes through the shared wire, which
186
+ prefers the OFFICIAL `anthropic` SDK and falls back to the same raw POST when the package is
187
+ absent. The shape of what comes back is unchanged, deliberately: `anthropic_send` normalises
188
+ both transports to `(status, body)`, so every line below this call is untouched.
189
+ """
190
+ global LAST_ANTHROPIC_TRANSPORT
191
+ import providers as _prov # noqa: PLC0415
192
+ # ⚠ NO TOOLS ON THIS PATH. The builder omits `tool_choice` for exactly that case, so there is
193
+ # nothing to strip here — the rule lives in `anthropic_request`, beside the thing it constrains.
194
+ req = _prov.anthropic_request(
195
+ model=model, key=os.environ[p["env"]].strip(), system=system,
196
+ messages=[{"role": "user", "content": user}], tools=[], max_tokens=300)
197
+ status, body, transport = _prov.anthropic_send(req, timeout=timeout)
198
+ LAST_ANTHROPIC_TRANSPORT = transport
199
+ if status >= 400 or status == 0:
200
+ # ⭐ THE SENTENCE COMES FROM THE LADDER, NOT FROM HERE. `refusal_sentence` already names the
201
+ # vendor and the action for every status R4 enumerated, so a bare "anthropic answered 402"
202
+ # (which is what this line used to say, and what the owner quoted back at us) cannot recur.
203
+ return "", _prov.refusal_sentence("anthropic", status, json.dumps(body)[:400]), None
204
+ body = body or {}
205
+ # ⛔ stop_reason FIRST. A safety refusal is a successful 200 with an EMPTY content list, so
206
+ # reading content[0] before this check turns a refusal into an IndexError inside a run.
207
+ # ⚠ A refusal IS billed and its body carries `usage`, so the body rides back on this branch too.
208
+ if body.get("stop_reason") == "refusal":
209
+ return "", "anthropic declined to answer this record", body
210
+ parts = [b.get("text") or "" for b in (body.get("content") or [])
211
+ if isinstance(b, dict) and b.get("type") == "text"]
212
+ if not parts:
213
+ return "", "anthropic returned no text", body
214
+ return "".join(parts), "", body
215
+
216
+
217
+ def decide(*, prompt, options, row, fields=(), label="Review", timeout=None, st=None, user=""):
218
+ """Pick this record's next stage. Returns `(choice, meta)`.
219
+
220
+ `choice` is "" whenever a person should decide — which is every failure mode there is.
221
+ `meta` carries `provider`, `model`, `reason` on success, and `problem` on refusal to answer.
222
+
223
+ ⭐ W35-T41 / C7 — `st` and `user` are the USAGE LEDGER's target. Both default to absent because
224
+ this function's caller is `automation_engine.ai_decide(rt, ...)`, in another lane's fence: it
225
+ HAS the runtime and does not pass it yet, so until it does, a review's tokens are counted as
226
+ unattributed and REPORTED by `GET /usage` rather than dropped. See `usage_ledger`'s header for
227
+ why they cannot be resolved implicitly (measured: a contextvar does not survive a FastAPI
228
+ dependency).
229
+ """
230
+ import usage_ledger # noqa: PLC0415
231
+ opts = [str(o) for o in (options or []) if str(o).strip()]
232
+ if not opts:
233
+ return "", {"problem": "the review offers no next stages"}
234
+ if not str(prompt or "").strip():
235
+ return "", {"problem": "the review has no prompt for the model to follow"}
236
+ live = ladder()
237
+ if not live:
238
+ return "", {"problem": "no AI provider is configured on this deployment"}
239
+ system = _instruction(prompt, opts, label)
240
+ user = "Here is the record:\n\n" + _record_text(row, fields)
241
+ tmo = float(timeout or TIMEOUT_SECONDS)
242
+ override = (os.environ.get("AIOS_AI_MODEL") or "").strip()
243
+ problems = []
244
+ for p in live:
245
+ model = override or p["model"]
246
+ body = None
247
+ try:
248
+ text, err, body = (_call_anthropic if p["shape"] == "anthropic" else _call_openai)(
249
+ p, model, system, user, tmo)
250
+ except Exception as e: # noqa: BLE001
251
+ text, err = "", f"{p['name']} failed: {type(e).__name__}"
252
+ # ⭐⭐ C7 — THE LEDGER LINE, BEFORE ANY BRANCH BELOW READS THE ANSWER.
253
+ # ⛔ IT IS WRITTEN ON EVERY OUTCOME THAT REACHED A PROVIDER, including a refusal and an
254
+ # unusable answer. A meter that books only successes reports a cheap week for a run that
255
+ # spent its budget being declined — and a declined call is billed. The only path that does
256
+ # NOT record is a transport that never reached the vendor (`body is None`), which spent
257
+ # nothing.
258
+ if body is not None:
259
+ ins, outs = usage_ledger.tokens_from(body)
260
+ usage_ledger.record("ai_review", p["name"], model, ins, outs,
261
+ total=usage_ledger.total_from(body), st=st, user=user)
262
+ if err:
263
+ problems.append(err)
264
+ continue # ladder: a dead provider degrades to the next one
265
+ choice, reason = _parse(text, opts)
266
+ if not choice:
267
+ # The provider ANSWERED and declined (or answered unusably). That is a decision about
268
+ # this record, not a fault in the provider, so it does NOT fall through to a more
269
+ # expensive one — the card goes to a human, which is what the model just asked for.
270
+ return "", {"provider": p["name"], "model": model,
271
+ "problem": reason or "the model did not choose one of the stages"}
272
+ return choice, {"provider": p["name"], "model": model, "reason": reason}
273
+ return "", {"problem": "; ".join(problems)[:300] or "no provider answered"}
274
+
275
+
276
+ # ══════════════════════════════════════════════════════════════════ the FLOW WRITER
277
+ # ⭐⭐ WAVE 33 · W33-T54/T55 (owner item 7, ruling R3) — A PROMPT BECOMES A DRAFT AUTOMATION.
278
+ #
279
+ # Owner, verbatim: *"lay out the foundation with custom tools we provide"* — and the tools are the
280
+ # action catalog, which has been machine-readable since wave 23. So this writes no new vocabulary:
281
+ # it hands the model the SAME `ACTION_CATALOG` the menu paints, the SAME `ACTION_REQUIRED` the
282
+ # runner blocks on, and the SAME trigger keys `clean_trigger` accepts, and asks for one JSON object
283
+ # in that vocabulary.
284
+ #
285
+ # ⛔ WHY IT LIVES IN `ai_review.py` RATHER THAN A NEW FILE. Two reasons, both about ownership. This
286
+ # is the one module in the API package that already holds an LLM ladder and its credentials, so a
287
+ # second one would be a second place a rotated key has to be noticed; and a new `routes_*.py` needs
288
+ # a `main.py` line from another lane (contract C2) to be reachable at all, which is how wave 23
289
+ # shipped three finished routers 404-dead behind green gates. The DOOR is a route on
290
+ # `routes_automation.py`, which is already mounted.
291
+ #
292
+ # ⛔⛔ CONTRACT C7 IS THE DESIGN, NOT A CHECK AT THE END: *"the flow writer may emit only kinds
293
+ # present in `ACTION_CATALOG`, and its output must pass `clean_actions` unchanged."* Both halves are
294
+ # enforced mechanically rather than asked for politely — the kind list is an `enum` in the schema
295
+ # the model fills, and the caller runs `clean_actions` and DIFFS the result. That diff matters
296
+ # because `clean_actions` has no disclosure channel (D-75): it drops a key it does not recognise
297
+ # and answers 200, so a draft accepted without the diff would show a person a flow that is not the
298
+ # flow the model described, with nothing anywhere saying so.
299
+ #
300
+ # ⚠ `chat` IS INJECTABLE, exactly as `routes_query._call_model`'s is, and it is the reason this door
301
+ # can be proven end to end with no API key and no spend. A gate that can only run where a
302
+ # credential exists is a gate that never runs.
303
+
304
+ #: Cerebras first, NOT the cheap-first order `ladder()` uses. Same reason `routes_query` inverts it:
305
+ #: this contract includes a REFUSAL, and refusing honestly ("I cannot build that from the steps you
306
+ #: have") is a model property. A cheaper rung answers an impossible request with a plausible flow,
307
+ #: which is worse than no flow. Override with `AIOS_FLOW_PROVIDER`.
308
+ FLOW_PROVIDER_ORDER = ("cerebras", "groq", "openrouter", "anthropic")
309
+ MAX_DRAFT_ACTIONS = 12 # a draft a person reads in one screen; the engine's own cap is 25
310
+ MAX_PROMPT_CHARS = 2000
311
+
312
+
313
+ def flow_providers(pin=None):
314
+ """The rungs usable here, in THIS module's refusal-first order. Empty = the feature is off.
315
+
316
+ ⭐ `pin` IS ASK D-18 (2026-08-18): the Agent chat's model toggle must configure something, and
317
+ the draft door used to read `prompt` off the body and nothing else — so the key the client sent
318
+ was accepted and dropped, and the picker was a control over nothing.
319
+ ⚠ AN UNKNOWN OR UNCONFIGURED PIN FALLS BACK TO THE LADDER rather than refusing. A model the
320
+ ladder stopped offering must not turn every later draft into an error; the caller is told which
321
+ rung actually answered, which is the honest half.
322
+ """
323
+ by_name = {p["name"]: p for p in PROVIDERS}
324
+ live = [n for n in FLOW_PROVIDER_ORDER
325
+ if n in by_name and (os.environ.get(by_name[n]["env"]) or "").strip()]
326
+ wanted = str(pin or os.environ.get("AIOS_FLOW_PROVIDER") or "").strip().lower()
327
+ if wanted and wanted in live:
328
+ return [by_name[wanted]]
329
+ return [by_name[n] for n in live]
330
+
331
+
332
+ def flow_schema(kinds, trigger_keys, table_keys):
333
+ """The tool schema the model fills — the catalog's OWN key lists as enums.
334
+
335
+ ⛔ `kind` AND `trigger` ARE ENUMS, not free strings, and that is C7's first half enforced by the
336
+ transport rather than by a check afterwards. A model asked for "any action name" invents
337
+ `send_slack_message` on a deployment that has no such kind, and the failure then surfaces as a
338
+ 400 from `clean_actions` carrying a sentence about a word the person never typed.
339
+ ⚠ `required` IS `["kind"]` ALONE, deliberately: a REFUSAL carries no name and no actions, and a
340
+ schema demanding them turns an honest refusal into a provider-side 400 that reads exactly like a
341
+ transport failure (`routes_query._spec_schema` carries the same note, for the same measured
342
+ reason).
343
+ """
344
+ return {
345
+ "type": "object",
346
+ "properties": {
347
+ "kind": {"type": "string", "enum": ["flow", "refused"],
348
+ "description": "refused = this cannot be built from the steps available"},
349
+ "refusal": {"type": "string",
350
+ "description": "when kind=refused: ONE plain sentence naming what is "
351
+ "missing, in the words a non-technical person would use"},
352
+ "name": {"type": "string", "description": "a short title for the automation"},
353
+ "trigger": {"type": "string", "enum": sorted(trigger_keys),
354
+ "description": "what starts this automation"},
355
+ "table": {"type": "string", "enum": sorted(table_keys),
356
+ "description": "the database whose records this flow walks, if any"},
357
+ "actions": {
358
+ "type": "array",
359
+ "maxItems": MAX_DRAFT_ACTIONS,
360
+ "items": {
361
+ "type": "object",
362
+ "properties": {
363
+ "kind": {"type": "string", "enum": sorted(kinds)},
364
+ "why": {"type": "string",
365
+ "description": "one short sentence: why this step is here"},
366
+ "config": {"type": "object", "additionalProperties": True,
367
+ "description": "the step's settings, using ONLY the keys named "
368
+ "for that kind in the system message"},
369
+ },
370
+ "required": ["kind"],
371
+ },
372
+ },
373
+ },
374
+ "required": ["kind"],
375
+ }
376
+
377
+
378
+ def flow_system_prompt(catalog, required, triggers, tables):
379
+ """What the model is told it may build with — DERIVED, never written down twice.
380
+
381
+ Every list here is the server's own: the catalog rows the menu paints, the required-key table
382
+ the runner blocks on, the trigger keys `clean_trigger` accepts, and this tenant's real databases
383
+ with their real columns. Nothing about the vocabulary is restated by hand, so a kind added to
384
+ the catalog is offered here on the same deploy and a kind removed stops being offered.
385
+ """
386
+ lines = ["You build small automations for a business tool. You are given the EXACT set of "
387
+ "steps this tool can perform. You may use nothing else.",
388
+ "", "THE STEPS YOU MAY USE:"]
389
+ # ⚠ BOTH HALVES, `key (phrase)`. The KEY is what the model must write into `config` and the
390
+ # PHRASE is the only human wording of that requirement anywhere — the one the runner's own
391
+ # refusal sentence is built from. Keys alone leave the model guessing what `field` means on a
392
+ # `web_read` (it is a column to write into, not a form field); phrases alone leave it guessing
393
+ # what to call the setting. Handing it one and hoping is how a draft comes back configured
394
+ # against a key the validator drops.
395
+ req = {str(k): [f"{key} ({phrase})" for phrase, key in v] for k, v in (required or {}).items()}
396
+ for row in catalog or []:
397
+ if not row.get("ready"):
398
+ continue
399
+ kind = str(row.get("kind") or "")
400
+ need = req.get(kind) or []
401
+ lines.append(f"- {kind}: {row.get('label')} - {row.get('detail') or ''}"
402
+ + (f" REQUIRED settings: {', '.join(need)}" if need else ""))
403
+ lines += ["", "WHAT CAN START AN AUTOMATION:"]
404
+ for t in triggers or []:
405
+ if t.get("planned") or not t.get("ready", True):
406
+ continue
407
+ lines.append(f"- {t.get('key')}: {t.get('label')}")
408
+ lines += ["", "THE DATABASES THIS PERSON HAS:"]
409
+ for t in (tables or [])[:40]:
410
+ cols = ", ".join(str(f.get("key")) for f in (t.get("fields") or [])[:25])
411
+ lines.append(f"- {t.get('key')} ({t.get('label')}): {cols or 'no columns yet'}")
412
+ if not tables:
413
+ lines.append("- (none - do not name a database)")
414
+ lines += [
415
+ "",
416
+ "RULES:",
417
+ "1. Use ONLY the step kinds listed above. If what is asked needs a step that is not there, "
418
+ "answer kind=refused and say plainly which capability is missing.",
419
+ "2. Fill in every REQUIRED setting you can from what the person told you. Leave one blank "
420
+ "rather than inventing a web address, a CSS selector or a column name.",
421
+ "3. A web address must start with http:// or https://.",
422
+ "4. To use a value from the record the flow is walking, write it as {{Column name}}.",
423
+ "5. Name a database only from the list above, by its key.",
424
+ "6. Keep it short. Fewer steps that work beat more steps that guess.",
425
+ ]
426
+ return "\n".join(lines)
427
+
428
+
429
+ def _flow_from_tool_call(obj):
430
+ """The model's tool arguments -> `(draft, refusal)`. A shape error is a refusal, never a crash."""
431
+ if not isinstance(obj, dict):
432
+ return None, "the assistant's answer could not be read"
433
+ if str(obj.get("kind") or "") == "refused":
434
+ return None, (str(obj.get("refusal") or "").strip()
435
+ or "the assistant could not build this from the steps available")
436
+ acts = obj.get("actions")
437
+ if not isinstance(acts, list) or not acts:
438
+ return None, ("the assistant did not produce any steps - try describing what should "
439
+ "happen, one action at a time")
440
+ out = []
441
+ for a in acts[:MAX_DRAFT_ACTIONS]:
442
+ if not isinstance(a, dict) or not str(a.get("kind") or "").strip():
443
+ continue
444
+ cfg = a.get("config")
445
+ out.append({"kind": str(a["kind"]).strip(),
446
+ "config": cfg if isinstance(cfg, dict) else {},
447
+ "why": str(a.get("why") or "").strip()[:200]})
448
+ if not out:
449
+ return None, "the assistant's steps could not be read"
450
+ return {"name": str(obj.get("name") or "").strip()[:80] or "New automation",
451
+ "trigger": str(obj.get("trigger") or "").strip(),
452
+ "table": str(obj.get("table") or "").strip(),
453
+ # ⛔⛔ THE TRUNCATION IS REPORTED, and it is a STANDING RULE that it must be (owner,
454
+ # 2026-08-12, W30/R6 second sentence): *"if there is lag or it can't be done, you need
455
+ # to explicitly tell me why and recommend a fix"* — **a silent truncation IS the
456
+ # violation, not the limit.** `acts[:MAX_DRAFT_ACTIONS]` above dropped everything past
457
+ # the twelfth and said nothing, so a model that answered with a twenty-step journey had
458
+ # eight steps deleted between the answer and the screen, with no key anywhere in the
459
+ # response naming them. `MAX_ACTIONS` is 20, so those steps were STORABLE — this
460
+ # ceiling is the draft door's own, which makes reporting it the whole obligation.
461
+ # ⚠ Found by the verifier that checked this ticket, not by a gate.
462
+ "asked": len([a for a in acts if isinstance(a, dict)]),
463
+ "actions": out}, ""
464
+
465
+
466
+ def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeout=None,
467
+ st=None, user="", model=None):
468
+ """A sentence -> `(draft, refusal_sentence, provider)`. ⛔ NOTHING IS SAVED HERE.
469
+
470
+ Exactly one of `draft` and `refusal_sentence` is truthy — the same contract
471
+ `web_agent.run_step` keeps, so a caller has no third case to get wrong.
472
+
473
+ ⭐ W35-T41 / C7 — `st`/`user` are the usage ledger's target, exactly as on `decide` above and for
474
+ the same reason: both of this function's callers (`automation_engine._ai_agent_plan` and
475
+ `routes_automation`'s draft door) are in lane D's fence. C7 says E adds the ledger line here and
476
+ D asserts it; the two keywords are what D has to pass for the line to be attributable.
477
+ """
478
+ # ⚠ DECLARED AT THE TOP OF THE FUNCTION, not beside the assignment inside the provider loop.
479
+ # It parses either way; a reader scanning for the declaration does not look inside a `for`.
480
+ global LAST_ANTHROPIC_TRANSPORT
481
+ import usage_ledger # noqa: PLC0415
482
+ text = str(prompt or "").strip()[:MAX_PROMPT_CHARS]
483
+ if not text:
484
+ return None, "type what you want the automation to do", None
485
+ kinds = sorted({str(r.get("kind")) for r in (catalog or []) if r.get("ready")})
486
+ if not kinds:
487
+ return None, "this deployment offers no automation steps to build with", None
488
+ trigger_keys = sorted({str(t.get("key")) for t in (triggers or [])
489
+ if t.get("key") and not t.get("planned")}) or ["manual"]
490
+ table_keys = sorted({str(t.get("key")) for t in (tables or []) if t.get("key")}) or [""]
491
+ tools = [{"type": "function", "function": {
492
+ "name": "build_automation",
493
+ "description": "Emit the automation, or refuse.",
494
+ "parameters": flow_schema(kinds, trigger_keys, table_keys)}}]
495
+ messages = [{"role": "system",
496
+ "content": flow_system_prompt(catalog, required, triggers, tables)},
497
+ {"role": "user", "content": text}]
498
+
499
+ if chat is not None:
500
+ # ⚠ THE INJECTED PATH IS THE PROVEN PATH. It runs the SAME parse and the SAME refusal
501
+ # branches as a live call; only the transport is replaced.
502
+ draft, refusal = _flow_from_tool_call(chat(messages, tools))
503
+ return draft, refusal, "injected"
504
+
505
+ provs = flow_providers(model)
506
+ if not provs:
507
+ # ⛔ SAY SO. An AI feature that silently does nothing is indistinguishable from one that was
508
+ # never built [[flag-shipped-without-its-writer]].
509
+ return None, ("the assistant is not configured on this deployment, so an automation cannot "
510
+ "be drafted from a description yet"), None
511
+ tmo = float(timeout or TIMEOUT_SECONDS)
512
+ problems = []
513
+ import providers as _prov
514
+ for p in provs:
515
+ # ⭐⭐ W36-T35 / R4 — THE OWNER QUOTED THIS LINE BACK AT US. It used to read
516
+ # `problems.append(f"{p['name']}: tool calls are not wired for this shape")` and `continue`,
517
+ # so the sentence *"anthropic: tool calls are not wired for this shape"* appeared under the
518
+ # Agent module verbatim. R4: *"Anthropic becomes the tool-calling path that always works."*
519
+ # It is wired now, through the ONE wire in `providers`, and it matters more than it looks:
520
+ # every side rung on this account is refusing today (cerebras 402, groq 404, openrouter
521
+ # 402), so without this branch the drafter cannot answer at all.
522
+ # ⚠ NO `effort` ON THIS DOOR — `output_config.effort` errors on Haiku 4.5, the tier this
523
+ # ladder runs. The parameter is the caller's to send, which is why the wire takes it.
524
+ # ⭐ D-346 (W37-T39): BOTH shapes now yield the SAME `(status, body)` pair, so every branch
525
+ # below reads one thing. The anthropic side gets there through `anthropic_send` (official
526
+ # SDK when installed, the identical raw POST when not); the openai side still posts here
527
+ # because there is no shared SDK across three different vendors on that wire.
528
+ global LAST_ANTHROPIC_TRANSPORT
529
+ try:
530
+ if p["shape"] == "anthropic":
531
+ req = _prov.anthropic_request(
532
+ model=p["model"], key=os.environ[p["env"]].strip(), system=None,
533
+ messages=messages, tools=tools, max_tokens=1500, tool_choice="required")
534
+ status, body, transport = _prov.anthropic_send(req, timeout=tmo)
535
+ LAST_ANTHROPIC_TRANSPORT = transport
536
+ else:
537
+ r = requests.post(p["url"], timeout=tmo,
538
+ headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}",
539
+ "Content-Type": "application/json"},
540
+ json={"model": p["model"], "messages": messages, "tools": tools,
541
+ "tool_choice": "required", "temperature": 0.1,
542
+ "max_tokens": 1500})
543
+ status = r.status_code
544
+ body = r.json() if (r.content and r.status_code == 200) else {"_text": r.text}
545
+ except Exception as e: # noqa: BLE001
546
+ problems.append(f"{p['name']}: {type(e).__name__}")
547
+ continue
548
+ if status != 200:
549
+ # ⛔ A SENTENCE, NOT `HTTP 402` (R4's third clause). The owner read the status codes off
550
+ # this very door. `refusal_sentence` also decides whether this was a CREDIT failure, and
551
+ # the memo makes the next turn SKIP the empty account instead of paying for it again.
552
+ # ⚠ The body is re-serialised for the substring test because the SDK path never had a
553
+ # `.text` — that is the one thing this normalisation costs, and it costs nothing else.
554
+ detail = json.dumps(body)[:600] if isinstance(body, dict) else str(body)[:600]
555
+ if _prov.is_credit_failure(status, detail):
556
+ _prov.mark_no_credit(p["name"])
557
+ problems.append(_prov.refusal_sentence(p["name"], status, detail))
558
+ continue
559
+ try:
560
+ body = body or {}
561
+ if p["shape"] == "anthropic":
562
+ _text, args, _refused = _prov.anthropic_read(body)
563
+ if _refused:
564
+ problems.append(f"{p['name']}: {_refused}")
565
+ continue
566
+ else:
567
+ calls = (((body.get("choices") or [{}])[0].get("message") or {})
568
+ .get("tool_calls") or [])
569
+ args = json.loads(calls[0]["function"]["arguments"]) if calls else None
570
+ except Exception as e: # noqa: BLE001
571
+ problems.append(f"{p['name']}: unreadable answer ({type(e).__name__})")
572
+ continue
573
+ # ⭐⭐ C7 — THE LEDGER LINE. Placed after the parse rather than before it so an UNREADABLE
574
+ # answer is not double-counted by the `continue` above... ⚠ which means an unreadable answer
575
+ # is NOT counted at all, and that is a deliberate, disclosed loss: a body this code cannot
576
+ # parse is a body whose `usage` it also cannot trust, and `body` is out of scope in that
577
+ # branch by construction. The 200-with-junk case is rare and named here rather than
578
+ # silently rounded to zero.
579
+ ins, outs = usage_ledger.tokens_from(body)
580
+ usage_ledger.record("automation_draft", p["name"], p["model"], ins, outs,
581
+ total=usage_ledger.total_from(body), st=st, user=user)
582
+ draft, refusal = _flow_from_tool_call(args)
583
+ if draft is None and isinstance(args, dict) and str(args.get("kind")) == "refused":
584
+ # ⛔ A REFUSAL IS AN ANSWER, NOT A FAULT, so it does NOT fall through to a more
585
+ # expensive rung — the model has just said this cannot be built. `decide()` takes the
586
+ # same posture for the same reason.
587
+ return None, refusal, p["name"]
588
+ if draft is not None:
589
+ return draft, "", p["name"]
590
+ problems.append(f"{p['name']}: {refusal}")
591
+ return None, ("; ".join(problems)[:300] or "no provider answered"), None
api/automation_engine.py CHANGED
The diff for this file is too large to render. See raw diff
 
api/connectors_tt.py CHANGED
@@ -1,568 +1,568 @@
1
- """connectors_tt.py — the TIKTOK connector (wave 29 · item 7 · DEBT D-9 · rulings R1 + R2).
2
-
3
- Everything in this file knows what a VENDOR's TikTok row looks like. Nothing in it knows what an
4
- automation is. That split is `connectors_ig.py`'s (wave 27 item 23) and it is the reason this file
5
- exists at all rather than another thousand lines inside the engine.
6
-
7
- ⛔ **EVERY VENDOR FIELD NAME HERE WAS PROBED, NOT GUESSED.** The whole schema — 40 profile / 43
8
- post / 17 comment fields, each with the vendor's own type, description and `pii` flag — was read
9
- live from `GET /datasets/{id}/metadata` for **$0.00** and written down in
10
- `.claude/wiki/waves/wave29/proto/tiktok-schema.md` (promoted to `tiktok-capture.md` at
11
- close-out). That document is the AUTHORITY: do not re-probe it, and do not invent a key. Where a
12
- name below reads through a candidate list it is because the vendor has two names for one fact
13
- (`biography`/`signature`, `region`/`country`), never because the name is uncertain.
14
-
15
- ⛔ **NOTHING HERE EVER AUTHENTICATES TO TIKTOK.** No login, no cookie, no account to get banned —
16
- public data through a supplier, exactly the rail `connectors_ig.py` states for Instagram. The
17
- vendor key is a key to a SUPPLIER.
18
-
19
- ⭐ **THE TRANSPORT IS `connectors_bd.py` — SHARED, VENDOR-NAMED, AND NO LONGER BORROWED FROM THE
20
- OTHER PLATFORM'S CONNECTOR** (WAVE 30 · T09, DEBT D-128). `bd_call`, `bd_scrape` and
21
- `bd_filter_start` take the dataset id as a PARAMETER — they are Bright Data's wire, not
22
- Instagram's — and re-implementing them here would be a second copy of the deferral handling, the
23
- truncation guard, the SSRF rail and the snapshot-progress reader, i.e. five places for one bug.
24
- Until wave 30 the code was right and the NAME was wrong: this file imported thirteen symbols from
25
- `connectors_ig`, which read as a dependency on Instagram and was really a dependency on a supplier.
26
- ⚠ **This file now imports ZERO names from `connectors_ig`, and a gate check asserts that**, because
27
- the sentence above is the kind that quietly stops being true.
28
-
29
- ⚠ **WHAT $0 COULD NOT BUY, so nobody reads this file as more measured than it is:**
30
- 1. the real ROW shape — `/metadata` describes a DATASET, and Instagram's rows carry undeclared
31
- envelope keys (`timestamp`, `input`) that no metadata call mentions;
32
- 2. that a declared field POPULATES — Bright Data's Instagram Reels *declares* `views: number`
33
- and delivers an account-grain wrong number (§4e). **Declared is not delivered**, and the one
34
- TikTok claim that matters most (`play_count`) is exactly a declaration.
35
- """
36
- from __future__ import annotations
37
-
38
- import automation_engine as engine
39
- # ⚠ T09 — FOUR NAMES CAME OFF THIS LIST AND NOTHING BROKE, which is the point of deriving an
40
- # import block from the AST both ways. `bd_call`, `bd_filter_start`, `bd_key` and `bd_ready` were
41
- # imported here under a comment claiming they were *"re-exported for the runners"*; no runner ever
42
- # read them off this module (the engine imports them from the transport itself), so they were four
43
- # lines of dependency nobody was paying for. `[[artifact-with-no-importer]]` in its smallest form.
44
- from connectors_bd import (
45
- _bd_first_url,
46
- _bd_flag,
47
- _bd_list,
48
- _bd_source_payload,
49
- _first,
50
- _ig_int,
51
- bd_scrape,
52
- )
53
-
54
- # ---------------------------------------------------------------------------------------------
55
- # THE DATASETS
56
- # ---------------------------------------------------------------------------------------------
57
- # ⚠ CATALOGUE PRESENCE IS NOT ENTITLEMENT — the same finding Instagram produced. `GET
58
- # /datasets/list` returned 1,735 rows of which 12 are TikTok; the three below answer 200 with a
59
- # full field list, and the two DISCOVERY halves answer **404 for our key**:
60
- # `gd_lj71gn6l68bz7y9hc` (posts by profile) and `gd_lilwhto81z415d9mdl` (posts by keyword).
61
- # ⇒ TikTok discovery routes through the PROFILES dataset's corpus filter, exactly as Instagram's
62
- # does. A ticket that reaches for a by-keyword endpoint is reaching for a 404.
63
- TT_DS_PROFILES = "gd_l1villgoiiidt09ci" # TikTok - Profiles. 40 fields, 152,000,000 records
64
- TT_DS_POSTS = "gd_lu702nij2f790tmv9h" # TikTok - Posts. 43 fields
65
- TT_DS_COMMENTS = "gd_lkf2st302ap89utw5k" # TikTok - Comments. 17 fields
66
-
67
- #: The vendor's two post-type tokens, verbatim from the dataset's own `ai_description`
68
- #: (*"strictly these two"*, video = 99.5% of rows). Ours are `image`/`video`/`carousel`.
69
- TT_POST_TYPE_VIDEO = "video"
70
- TT_POST_TYPE_CONTENT = "content"
71
-
72
- #: What a TikTok profile URL looks like, for the runner that has a handle and needs a URL. Kept
73
- #: beside the dataset ids because it is the same class of vendor fact.
74
- #: ⚠ ONE SPELLING. `platform/core/user_tables.profile_url(handle, 'tiktok')` builds the identical
75
- #: string from `_PROFILE_RULES` (contract C2, E's half) — this exists for the connector's own
76
- #: batch calls, and the gate asserts the two agree rather than trusting that they do.
77
- TT_PROFILE_URL = "https://www.tiktok.com/@{handle}"
78
-
79
-
80
- def tt_profile_url(handle):
81
- """`nurilab` → `https://www.tiktok.com/@nurilab`. `''` for a blank handle, never a bare `@`."""
82
- h = str(handle or "").strip().lstrip("@")
83
- return TT_PROFILE_URL.format(handle=h) if h else ""
84
-
85
-
86
- # ---------------------------------------------------------------------------------------------
87
- # THE FIELD MAPS
88
- # ---------------------------------------------------------------------------------------------
89
- # Each function turns ONE vendor row into the cell dict for one of the `ut_tt_*` schemas declared
90
- # in `automation_engine`. The rules they all obey, stated once:
91
- #
92
- # * **BLANK MEANS NOT READ, NEVER "THEY HAVE NONE".** A key the vendor did not send is OMITTED,
93
- # so a later, richer pull fills it instead of being overwritten by this one's silence. `_first`
94
- # returns `None` (never 0) when nothing matches, which is what makes that possible.
95
- # * **A zero from the vendor is a MEASUREMENT and survives.** (Instagram's `_ig_zero_is_blank`
96
- # rule is scoped to a paid rung whose zeros were proven fictional; nothing here has earned it.)
97
- # * **Every unpromoted vendor key stays whole in `source_payload`.** A schema addition on the
98
- # vendor's side is preserved rather than silently discarded while our column model catches up.
99
- # * **Nothing here writes a session token.** `tt_chain_token`, `secu_id` (~85% null), `short_id`
100
- # (100% null in the sample), `ftc` (100% null) and `relation` are deliberately unmapped; they
101
- # ride in `source_payload` where they make no claim.
102
-
103
-
104
- def _tt_str(node, *names):
105
- """The first non-empty string among `names`, or None when the vendor sent nothing.
106
-
107
- ⛔ `None`, NOT `""`. The callers below drop `None` keys, which is what keeps a blank honest —
108
- an empty string written into a cell claims "we looked and it is empty".
109
- """
110
- v = _first(node, *names)
111
- if v is None:
112
- return None
113
- s = str(v).strip()
114
- return s or None
115
-
116
-
117
- def _tt_pct(node, *names):
118
- """A vendor 0–1 engagement fraction → our stored 0–100 percentage, or None.
119
-
120
- ⛔ THE ×100 IS NOT COSMETIC (wave 26, amendment C1-a). Our `pct` renderer appends the sign to
121
- the STORED number, so writing the vendor's raw 0.0656 would print a 6.6% creator as `0.0%` —
122
- measured on the Instagram side, and TikTok sends the same shape on all three of its rates.
123
- """
124
- v = _first(node, *names)
125
- if v is None:
126
- return None
127
- out = engine._pct100(v)
128
- return out or None
129
-
130
-
131
- def _tt_day(node, *names):
132
- """A vendor stamp → `YYYY-MM-DD`, or None. Our `date` columns store a day."""
133
- v = _first(node, *names)
134
- if v is None:
135
- return None
136
- return engine._day(v) or None
137
-
138
-
139
- def _drop_blanks(row):
140
- """The one place a mapped row loses its `None`s — see the BLANK MEANS NOT READ rule above."""
141
- return {k: v for k, v in row.items() if v is not None and v != ""}
142
-
143
-
144
- def normalize_profile(node, handle=""):
145
- """A TikTok Profiles row → the `ut_tt_profile` / `ut_tt_snapshots` cell shape.
146
-
147
- ⚠ TWO FIELDS READ THROUGH A CANDIDATE PAIR, and both pairs are the vendor's, not a guess:
148
- * `biography` is PRIMARY and `signature` the FALLBACK — the probe measured `signature`
149
- populated on 85% of rows and they carry the same text;
150
- * `region` is PRIMARY and `country` the FALLBACK — `region` is the one with a documented
151
- two-letter-ISO description, `country` has no description at all.
152
- ⚠ `videos_count` is mapped to `posts_count` with a caveat recorded rather than hidden: its
153
- `ai_description` ranges 1-89, so it may be a WINDOW rather than a lifetime total. It is the
154
- only count of its kind the dataset offers.
155
- """
156
- node = node if isinstance(node, dict) else {}
157
- account = _tt_str(node, "account_id") or str(handle or "").strip().lstrip("@")
158
- return _drop_blanks({
159
- "platform": engine.PLATFORM_TIKTOK,
160
- "handle": account,
161
- "full_name": _tt_str(node, "nickname"),
162
- "tt_id": _tt_str(node, "id"),
163
- "profile_url": _tt_str(node, "url") or (tt_profile_url(account) or None),
164
- "bio": _tt_str(node, "biography", "signature"),
165
- "external_url": _bd_first_url(_first(node, "bio_link")),
166
- "verified": _bd_flag(node, "is_verified"),
167
- "is_private": _bd_flag(node, "is_private"),
168
- # ⚠ APPROXIMATE, and the word is the vendor's: `is_commerce_user` has *"many null values"*
169
- # on their own description. It is the closest thing TikTok has to Instagram's
170
- # `is_business_account`, and `_bd_flag` writes nothing at all when the key is absent — so
171
- # the approximation only ever fills a cell the vendor actually answered.
172
- "is_business": _bd_flag(node, "is_commerce_user"),
173
- "followers": _ig_int(_first(node, "followers")),
174
- "following": _ig_int(_first(node, "following")),
175
- "posts_count": _ig_int(_first(node, "videos_count")),
176
- # ⚠ `likes` on a PROFILE row is likes RECEIVED across the account's videos (18-110,200, no
177
- # nulls). It is not a post-level number and it is not our `likes` column, which is why it
178
- # is stored under a different name.
179
- "likes_received": _ig_int(_first(node, "likes")),
180
- "avg_engagement": _tt_pct(node, "awg_engagement_rate"),
181
- "like_engagement": _tt_pct(node, "like_engagement_rate"),
182
- "comment_engagement": _tt_pct(node, "comment_engagement_rate"),
183
- "country_code": _tt_str(node, "region", "country"),
184
- "region": _tt_str(node, "region"),
185
- "predicted_lang": _tt_str(node, "predicted_lang"),
186
- # ⚠ ACCOUNT AGE, NOT A MEASUREMENT STAMP — `create_time` on a profile is when the ACCOUNT
187
- # was made. TikTok stamps nothing with "when this number was true", exactly like Instagram,
188
- # which is why the append law dates a snapshot by when WE read it.
189
- "account_created_at": _tt_day(node, "create_time"),
190
- "source_payload": _bd_source_payload(node),
191
- })
192
-
193
-
194
- def tt_post_type(node):
195
- """A TikTok post row → one of OUR three type options, or None.
196
-
197
- ⛔ THE FIRST OF THE PROBE DOC'S TWO NAMED BLOCKERS. The vendor's vocabulary is `"video"` /
198
- `"content"`; ours is `image` / `video` / `carousel` and has no `"content"`. Writing the
199
- vendor's token would fail `_clean_field`'s option check on the way in and would put an
200
- untranslated API word in front of a user on the way out.
201
-
202
- So: `video` is `video`, and `content` — TikTok's photo-mode post — is `image`, EXCEPT when the
203
- row carries more than one `carousel_images` entry, which is what a carousel IS on either
204
- network. ⚠ The multi-image branch is decided from the IMAGES, never from the type token: the
205
- token cannot express it, so inferring `carousel` from the word would be inventing a fact.
206
- ⚠ An UNKNOWN token returns None rather than defaulting to `video` (99.5% of rows are video, and
207
- that is exactly what would make the wrong default invisible).
208
- """
209
- raw = str((node or {}).get("post_type") or "").strip().lower()
210
- if raw == TT_POST_TYPE_VIDEO:
211
- return "video"
212
- if raw != TT_POST_TYPE_CONTENT:
213
- return None
214
- images = (node or {}).get("carousel_images")
215
- return "carousel" if isinstance(images, list) and len(images) > 1 else "image"
216
-
217
-
218
- def normalize_post(node):
219
- """A TikTok Posts row → the `ut_tt_posts` cell shape. None when it carries no identity.
220
-
221
- ⛔ THE SECOND NAMED BLOCKER, RESOLVED HERE AND NOWHERE ELSE: `play_count` is ONE number and
222
- Instagram's schema has TWO columns for it (`plays` and `views`). On TikTok they are the same
223
- fact — `play_count` IS the count TikTok displays under a video — so it maps to `views` and
224
- `ut_tt_posts` HAS NO `plays` COLUMN. Copying one vendor number into two of our columns would
225
- manufacture a second measurement that a rollup could average or double-count, which is a worse
226
- outcome than the missing column it would paper over.
227
-
228
- ⚠ `shortcode` reads `shortcode` then `post_id`: both are 19-digit numerics on this dataset and
229
- the probe measured them as the same shape. That equality is what lets the comments→posts link
230
- join with NO normaliser, which the Instagram side never had.
231
- ⚠ `num_share_count` (a number) is preferred over `share_count` (typed TEXT by the vendor).
232
- ⚠ `commerce_info` is a business/commerce LOCATION per its own description — cities and
233
- countries. It is NOT a paid-partnership flag, and nothing in this dataset is: TikTok declares
234
- no equivalent, so `paid_partnership`/`partner` have no column on this family at all.
235
- """
236
- node = node if isinstance(node, dict) else {}
237
- shortcode = _tt_str(node, "shortcode", "post_id")
238
- if not shortcode:
239
- return None
240
- return _drop_blanks({
241
- "platform": engine.PLATFORM_TIKTOK,
242
- "shortcode": shortcode,
243
- # ⛔⛔ `account_id`, NOT `profile_username` — MEASURED on a real Posts row 2026-08-12.
244
- # The vendor's `profile_username` is the DISPLAY NAME (`"Dina"`), while `account_id` is the
245
- # @handle (`"d1na_th"`) — the same field `normalize_profile` already reads for `handle`, so
246
- # one name means one thing across both corpora. Reading the display name silently broke the
247
- # only join this table has: `ut_tt_posts.influencer_key` -> `ut_tt_profile.handle` matched
248
- # NOTHING, so a person could not filter posts by creator and a rollup would count zero.
249
- # ⚠ NO FALLBACK TO `profile_username`, deliberately. It is not a degraded handle, it is a
250
- # different fact, and filling a join key with it is worse than leaving it blank — a blank
251
- # is visibly missing, a display name looks like an answer [[one-question-two-normalizers]].
252
- # The URL is the honest second source: it carries the handle by construction.
253
- "influencer_key": (_tt_str(node, "account_id")
254
- or tt_handle(_tt_str(node, "profile_url", "url") or "") or None),
255
- "posted_at": _tt_day(node, "create_time"),
256
- "type": tt_post_type(node),
257
- "caption": _tt_str(node, "description"),
258
- "url": _tt_str(node, "url"),
259
- "hashtags": _bd_list(node, "hashtags"),
260
- "tagged_location": _tt_str(node, "commerce_info"),
261
- "views": _ig_int(_first(node, "play_count")),
262
- "likes": _ig_int(_first(node, "digg_count")),
263
- "comments": _ig_int(_first(node, "comment_count")),
264
- "shares": _ig_int(_first(node, "num_share_count")),
265
- "saves": _ig_int(_first(node, "collect_count")),
266
- "video_duration": _ig_int(_first(node, "video_duration")),
267
- "source_payload": _bd_source_payload(node),
268
- })
269
-
270
-
271
- def normalize_comment(node):
272
- """A TikTok Comments row → the `ut_tt_comments` cell shape. None without a comment id.
273
-
274
- ⚠ THE NAME COLLISION, RESOLVED: TikTok's `replies` is an ARRAY of reply objects and OUR
275
- `replies` column is an INT count. The count comes from `num_replies`; the array stays whole in
276
- `source_payload`. Reading the array's length instead would be a second, disagreeing answer to
277
- a question the vendor already answers — and it would disagree, because a page of replies is not
278
- all of them.
279
- ⚠ `date_created` is typed `date` by this vendor, unlike Instagram's `comment_date` which is
280
- text and needs defensive parsing. It still goes through `_tt_day` — one date path, so a vendor
281
- that changes its mind cannot change ours.
282
- ⚠ The comment TEXT and every identifiable commenter field (`commenter_user_name` is flagged
283
- PII) stay in `source_payload` and are promoted to no column, which is the same posture the
284
- Instagram comment schema takes for the same D-24 reason.
285
- """
286
- node = node if isinstance(node, dict) else {}
287
- comment_key = _tt_str(node, "comment_id")
288
- if not comment_key:
289
- return None
290
- return _drop_blanks({
291
- "platform": engine.PLATFORM_TIKTOK,
292
- "comment_key": comment_key,
293
- "shortcode": _tt_str(node, "post_id"),
294
- # ⭐⭐ OWNER RULING 2026-08-12: the comment's CONTENT gets a column. `comment_text` is the
295
- # vendor's own key and `comment_text_only` its stripped variant (the probe recorded both);
296
- # primary first, so a row carrying the rich form is not silently served the plain one.
297
- # ⛔ The commenter's identity is deliberately NOT promoted — see `TT_COMMENT_FIELDS`.
298
- "text": _tt_str(node, "comment_text", "comment_text_only"),
299
- "commented_at": _tt_day(node, "date_created"),
300
- "likes": _ig_int(_first(node, "num_likes")),
301
- "replies": _ig_int(_first(node, "num_replies")),
302
- "source_payload": _bd_source_payload(node),
303
- })
304
-
305
-
306
- #: ⭐ The three maps, addressable by name — so a gate (and the runners in T04-T06) can walk them
307
- #: rather than naming three functions, and so adding a fourth dataset is one entry.
308
- TT_NORMALIZERS = {
309
- "tt_profile": normalize_profile,
310
- "tt_post_metrics": normalize_post,
311
- "tt_comments": normalize_comment,
312
- }
313
-
314
-
315
- # ---------------------------------------------------------------------------------------------
316
- # THE FETCH — wave 30 · W30-T08 (carrying wave-29's dropped T05)
317
- # ---------------------------------------------------------------------------------------------
318
-
319
- def tt_handle(url):
320
- """A TikTok profile URL **or** a bare handle → the handle. `''` when it is neither.
321
-
322
- Deliberately permissive about the input and strict about the output, because the two callers
323
- hand it different things: an automation stores whatever a person typed in the profile column
324
- (`@nurilab`, `nurilab`, or the full URL), while the discovery runner already holds a clean
325
- `account_id`. One normaliser, so a row found by discovery and a row typed by hand cannot
326
- resolve to two different handles.
327
- """
328
- s = str(url or "").strip()
329
- if not s:
330
- return ""
331
- if "tiktok.com" in s.lower():
332
- # Everything after the first `@`, up to the next path segment or query.
333
- tail = s.split("@", 1)[1] if "@" in s else ""
334
- s = tail.split("/")[0].split("?")[0].split("#")[0]
335
- s = s.strip().lstrip("@").strip()
336
- # A handle is the vendor's `account_id` shape: alphanumerics, dots and underscores.
337
- return s if s and all(c.isalnum() or c in "._" for c in s) else ""
338
-
339
-
340
- def tt_post_urls(node, limit=0):
341
- """⭐ WAVE 30 · T10 — the profile row's own post permalinks, newest-first as the vendor sends.
342
-
343
- ⛔ THIS IS WHY TIKTOK POST CAPTURE COSTS NO EXTRA DISCOVERY. `top_videos` rides the PROFILE row
344
- we have already bought, so the posts read is a scrape of links we hold, never a search for them.
345
- The two TikTok DISCOVERY datasets (posts-by-profile, posts-by-keyword) are **404 for our key**,
346
- so a design that reached for either would not merely be dearer, it would not work.
347
-
348
- ⛔⛔ CORRECTED 2026-08-12 — THIS DOCSTRING USED TO SAY *"the probe MEASURED `top_videos` as an
349
- array of video permalinks, NO empties"*, AND THAT SENTENCE IS WHAT SHIPPED THE BUG. The probe
350
- read `/datasets/{id}/metadata` — a DATASET description — and the phrase quoted was the field's
351
- `ai_description`, not an observation of a row. The real row sends **dicts keyed `video_url`**
352
- (measured below), so the reader built against the quoted sentence found nothing, forever, in
353
- silence. ⭐ The transferable half: *"the probe measured X"* and *"the probe read a declaration
354
- of X"* are different claims, and prose cannot be told apart by a reader downstream — which is
355
- why the correction names the method, not just the value.
356
-
357
- ⚠ `top_posts_data` is deliberately NOT read: the probe calls it *"a thin dup of `top_videos`"*,
358
- and preferring whichever happened to be longer is how one creator's window silently differs
359
- from another's.
360
- ⚠ `limit <= 0` means "everything the row carried". The CAP IS THE CALLER'S — `config.maxPosts`,
361
- validated 1..12 — and it is applied here rather than after the scrape so an unwanted post is
362
- never bought. [[a-constant-two-features-share]]: the 12 is the vendor's measured profile window,
363
- not a number this function may invent.
364
- """
365
- raw = (node or {}).get("top_videos")
366
- out = []
367
- for item in raw if isinstance(raw, list) else []:
368
- # ⛔⛔ MEASURED ON A REAL ROW 2026-08-12, AND IT IS NOT WHAT THE SCHEMA SAID.
369
- # `top_videos` is NOT an array of permalink strings. One paid TikTok Profiles scrape of a
370
- # live handle returns **19 DICTS**, keyed
371
- # `video_url · video_id · playcount · diggcount · commentcount · share_count ·
372
- # favorites_count · create_date · cover_image`.
373
- # The docstring above cites the $0 probe as having "measured" permalinks — it had not, and
374
- # could not: `/datasets/{id}/metadata` describes a DATASET, and the probe's own verdict says
375
- # so in terms (*"what $0 cannot buy … the real ROW shape … that a declared field
376
- # POPULATES"*, `wave29/proto/tiktok-schema.md`). This is the SECOND time this vendor's
377
- # declaration has diverged from its delivery on this exact axis; BD's IG Reels `views` was
378
- # the first. [[reachable-is-not-the-same-as-built]]
379
- # ⇒ The consequence, live: `TT_DS_POSTS` was never reached, because this returned an EMPTY
380
- # list on every real profile — post capture could not have worked for anybody, and the
381
- # T10 gate stayed green because its canned fixture encoded the DECLARED shape. A fixture
382
- # written from a schema tests the schema.
383
- # ⚠ `video_url` is FIRST because it is the key the vendor actually sends; `url` is kept
384
- # because it costs nothing and is what a future corpus revision would most likely use. The
385
- # bare-string branch stays for the same reason — this widens what is ACCEPTED and invents
386
- # nothing: a shape that yields no `http…` value still degrades to "no posts", exactly as
387
- # before, rather than to a URL built out of a guess.
388
- # ⚠ `top_posts_data` is STILL not read (it carries `post_url` and would work): preferring
389
- # whichever array happened to be longer is how one creator's window silently differs from
390
- # another's, and that reasoning is unchanged by this correction.
391
- if isinstance(item, dict):
392
- u = str(item.get("video_url") or item.get("url") or "").strip()
393
- else:
394
- u = str(item or "").strip()
395
- if u.lower().startswith("http") and u not in out:
396
- out.append(u)
397
- return out[:limit] if limit and limit > 0 else out
398
-
399
-
400
- def pull_posts_tt(post_urls, log=print, deferred=None):
401
- """The TikTok Posts dataset for a list of permalinks → `(rows, note)`, already normalised.
402
-
403
- ⚠ ONE CALL FOR THE WHOLE WINDOW. `bd_scrape` has always taken a list, and the Instagram side
404
- measured what happens when a caller forgets: 25 records, one billed snapshot each, a walk still
405
- running at 67 minutes. Nothing here loops per URL.
406
- """
407
- urls = [str(u) for u in (post_urls or []) if str(u or "").strip()]
408
- if not urls:
409
- return [], ""
410
- rows, note = bd_scrape(TT_DS_POSTS, urls, deferred=deferred)
411
- if note:
412
- log(f"[aios-tt] posts: {note}")
413
- return [], note
414
- out = [r for r in (normalize_post(n) for n in rows) if r]
415
- return out, ""
416
-
417
-
418
- def pull_comments_tt(post_urls, log=print, deferred=None):
419
- """The TikTok Comments dataset for a list of POST permalinks → `(rows, note)`, normalised.
420
-
421
- ⛔ THE MOST EXPENSIVE THING THIS PRODUCT BUYS, and the reason `commentMetrics` defaults OFF on
422
- both networks: a comments scrape ingests identifiable third parties who never entered anybody's
423
- list (D-24). The mapper already keeps every commenter field in `source_payload` and promotes
424
- none of them to a column; this function adds no new exposure, it just has to be asked for.
425
- """
426
- urls = [str(u) for u in (post_urls or []) if str(u or "").strip()]
427
- if not urls:
428
- return [], ""
429
- rows, note = bd_scrape(TT_DS_COMMENTS, urls, deferred=deferred)
430
- if note:
431
- log(f"[aios-tt] comments: {note}")
432
- return [], note
433
- out = [r for r in (normalize_comment(n) for n in rows) if r]
434
- return out, ""
435
-
436
-
437
- #: ⭐⭐ WAVE 30 · D-156 — THE MEDIA DATASETS, AS A SET, SO THE HAND-OFF CAN FILTER ON IDENTITY.
438
- #: `_media_deferrals` uses this to lift ONLY posts/comments snapshots out of the local deferral
439
- #: list. That is what makes it structurally impossible to file a PROFILE snapshot in the engine's
440
- #: metric queue — the defect a draft of T10 shipped and A-39 booked as "the wrong fix is worse
441
- #: than the gap". A membership test cannot be got wrong by a later edit the way `if` order can.
442
- TT_MEDIA_DATASETS = (TT_DS_POSTS, TT_DS_COMMENTS)
443
-
444
-
445
- def _media_deferrals(deferred):
446
- """The POSTS/COMMENTS entries of a `bd_scrape` deferral list — never the profile's.
447
-
448
- ⚠ The engine, not this module, decides what a deferral MEANS: it stamps `kind` and the
449
- handle and files it. TikTok needs no `_tag_metric_deferrals` twin because one dataset is one
450
- kind here, so the id already carries everything a mapper choice depends on — and importing
451
- Instagram's tagger is not available anyway (W30-T09 gates ZERO `from connectors_ig` lines).
452
- """
453
- out = []
454
- for d in deferred or []:
455
- if isinstance(d, dict) and str(d.get("datasetId") or "") in TT_MEDIA_DATASETS:
456
- out.append(dict(d))
457
- return out
458
-
459
-
460
- def pull_profile_tt(url, log=print, pending_profile=None, prefetch=None,
461
- max_posts=0, post_metrics=False, comment_metrics=False):
462
- """ONE TikTok profile from the vendor. Same return contract as `pull_profile`.
463
-
464
- `{state, profile, posts, comments, via, note}` with `state ∈ ok | partial | blocked | error`,
465
- so the engine's enrich branch treats every network identically and no caller learns a new
466
- shape.
467
-
468
- ⭐ WAVE 30 · T10 — POSTS AND COMMENTS ARE REAL NOW, AND BOTH DEFAULT OFF, exactly as Instagram's
469
- do. `post_metrics` scrapes the profile row's own `top_videos` permalinks (see `tt_post_urls` —
470
- no discovery call, because both TikTok discovery datasets 404 for our key); `comment_metrics`
471
- then scrapes the comments of the posts that came back. ⚠ COMMENTS REQUIRE POSTS by construction
472
- rather than by a rule: their input IS a post permalink, so asking for comments with post capture
473
- off is a request with no subject, and it returns none instead of quietly buying posts nobody
474
- asked for.
475
-
476
- ⛔ `partial` IS THE SUCCESS STATE WHENEVER NO MEDIA WAS READ, and that is deliberate rather than
477
- pessimistic. The Instagram contract reads `ok` only when identity AND media both landed
478
- (`pull_profile_bd`: *"identity without media is still partial ... a run that wrote a follower
479
- count and no posts must not paint green over a posts table that did not grow"*). So: posts not
480
- ASKED for → `partial`, saying so; posts asked for and landed → `ok`; asked for and none came →
481
- `partial` with the vendor's reason. The state answers "did this pull deliver what it went for",
482
- never "did the function finish".
483
-
484
- ⚠ **NO FREE RUNG, AND NO FALLBACK CHAIN.** Instagram's `pull_profile` drops to Apify when the
485
- paid rung refuses; `providers.DEFAULT_CHAINS["tt_profile"]` is deliberately single-provider,
486
- with its own note explaining that a multi-provider chain is a promise something walks it and
487
- that nothing walks Instagram's second name today either. So a refusal here is final, and it
488
- says so instead of implying a retry somewhere.
489
- """
490
- handle = tt_handle(url)
491
- if not handle:
492
- return {"state": "error", "profile": {}, "posts": [], "comments": [], "via": "",
493
- "note": f"{url!r} is not a TikTok profile URL or handle"}
494
-
495
- # ⭐ THE BATCH FAST PATH, same shape as the Instagram side: `prefetch` is `{handle: node}` from
496
- # one multi-URL scrape covering a whole selection. A hit is a vendor round trip that does not
497
- # happen; a miss falls through to the single-URL call below.
498
- cached = prefetch.get(handle) if isinstance(prefetch, dict) else None
499
- _deferred = []
500
- if isinstance(cached, dict) and cached:
501
- rows, note = [cached], ""
502
- else:
503
- rows, note = bd_scrape(TT_DS_PROFILES, [tt_profile_url(handle)], deferred=_deferred)
504
-
505
- node = rows[0] if rows else {}
506
- profile = normalize_profile(node, handle) if node else {}
507
- # ⛔ THE READABILITY TEST IS `followers`/`following`, NOT "did we get a dict". `normalize_profile`
508
- # drops blanks, so an unreadable row still returns `{"platform": …, "handle": …}` — truthy, and
509
- # carrying nothing anybody asked for. The Instagram rung tests exactly this pair for exactly
510
- # this reason, and answering "0 followers" instead is the failure it exists to prevent.
511
- unreadable = profile.get("followers") is None and profile.get("following") is None
512
- if note or unreadable:
513
- # ⭐ THE DEFERRAL IS HANDED OVER RATHER THAN DISCARDED. A snapshot the vendor is still
514
- # building HAS ALREADY BEEN PAID FOR; dropping its id bills again on the next run for the
515
- # same record. That was live on the Instagram profile path until 2026-08-09 — measured on
516
- # nurilab as two runs, two fresh snapshots, both abandoned — and it is not being
517
- # reintroduced here by omission.
518
- if isinstance(pending_profile, list):
519
- for d in _deferred:
520
- pending_profile.append({**d, "kind": "profile", "influencer": handle})
521
- why = note or ("the scrape answered, but no follower/following counts were readable in it "
522
- "(the field names may have moved - see tiktok-capture.md)")
523
- return {"state": "blocked", "profile": {}, "posts": [], "comments": [], "via": "brightdata",
524
- "deferredProfile": [d.get("snapshotId") for d in _deferred],
525
- "note": why}
526
-
527
- # --- W30-T10: THE MEDIA, ONLY WHEN IT WAS ASKED FOR. ------------------------------------
528
- if not post_metrics:
529
- return {"state": "partial", "profile": profile, "posts": [], "comments": [],
530
- "via": "brightdata",
531
- "note": note or "profile read; post capture is off for this step"}
532
- urls = tt_post_urls(node, limit=max_posts)
533
- if not urls:
534
- # ⚠ NOT AN ERROR AND NOT A RETRY. A creator with no `top_videos` has nothing to buy, and
535
- # saying so is what stops the next run paying to be told the same thing.
536
- return {"state": "partial", "profile": profile, "posts": [], "comments": [],
537
- "via": "brightdata",
538
- "note": note or "profile read; this account's row carried no post links"}
539
- posts, p_note = pull_posts_tt(urls, log=log, deferred=_deferred)
540
- comments, c_note = ([], "")
541
- if comment_metrics and posts:
542
- # The comments dataset is keyed on a POST permalink, so it reads the posts we just bought —
543
- # `url` from the mapper, never the profile's raw array, so a post the posts scrape refused
544
- # is not silently asked about again one rung later.
545
- comments, c_note = pull_comments_tt([p.get("url") for p in posts if p.get("url")],
546
- log=log, deferred=_deferred)
547
- # ⭐⭐ WAVE 30 · D-156 — THE MEDIA DEFERRALS ARE HANDED BACK, and the shape of the hand-off is
548
- # the whole lesson. An earlier draft of T10 appended every `_deferred` entry to
549
- # `pending_profile` tagged `kind: "profile"`. By the time control reaches here a PROFILE
550
- # deferral is impossible — the profile branch above returns `blocked` on any note — so **every
551
- # id fanned out that way was a POSTS or COMMENTS snapshot in the PROFILE queue**, whose
552
- # collector writes preset profile cells onto somebody's record from post rows. The engine keeps
553
- # the two queues apart deliberately (`_pending_profile_tasks` vs `_pending_metric_tasks`).
554
- # ⇒ So this returns them under their OWN key, filtered by dataset identity
555
- # (`_media_deferrals`), and the engine files them in the metric queue with the handle it
556
- # already holds. Returning rather than appending also keeps the queue's vocabulary out of a
557
- # connector: this module knows which CORPUS deferred, never what the engine calls it.
558
- # ⚠ `deferredMedia` rides BOTH returns on purpose. The empty-posts case is the one that
559
- # matters most — that is exactly the run where the vendor took too long, so a caller reading
560
- # the ids only from the success path would lose every batch it actually paid for.
561
- deferred_media = _media_deferrals(_deferred)
562
- if not posts:
563
- return {"state": "partial", "profile": profile, "posts": [], "comments": [],
564
- "via": "brightdata", "deferredMedia": deferred_media,
565
- "note": p_note or note or "profile read; the post source returned nothing"}
566
- return {"state": "ok", "profile": profile, "posts": posts, "comments": comments,
567
- "via": "brightdata", "deferredMedia": deferred_media,
568
- "note": c_note or note or ""}
 
1
+ """connectors_tt.py — the TIKTOK connector (wave 29 · item 7 · DEBT D-9 · rulings R1 + R2).
2
+
3
+ Everything in this file knows what a VENDOR's TikTok row looks like. Nothing in it knows what an
4
+ automation is. That split is `connectors_ig.py`'s (wave 27 item 23) and it is the reason this file
5
+ exists at all rather than another thousand lines inside the engine.
6
+
7
+ ⛔ **EVERY VENDOR FIELD NAME HERE WAS PROBED, NOT GUESSED.** The whole schema — 40 profile / 43
8
+ post / 17 comment fields, each with the vendor's own type, description and `pii` flag — was read
9
+ live from `GET /datasets/{id}/metadata` for **$0.00** and written down in
10
+ `.claude/wiki/waves/wave29/proto/tiktok-schema.md` (promoted to `tiktok-capture.md` at
11
+ close-out). That document is the AUTHORITY: do not re-probe it, and do not invent a key. Where a
12
+ name below reads through a candidate list it is because the vendor has two names for one fact
13
+ (`biography`/`signature`, `region`/`country`), never because the name is uncertain.
14
+
15
+ ⛔ **NOTHING HERE EVER AUTHENTICATES TO TIKTOK.** No login, no cookie, no account to get banned —
16
+ public data through a supplier, exactly the rail `connectors_ig.py` states for Instagram. The
17
+ vendor key is a key to a SUPPLIER.
18
+
19
+ ⭐ **THE TRANSPORT IS `connectors_bd.py` — SHARED, VENDOR-NAMED, AND NO LONGER BORROWED FROM THE
20
+ OTHER PLATFORM'S CONNECTOR** (WAVE 30 · T09, DEBT D-128). `bd_call`, `bd_scrape` and
21
+ `bd_filter_start` take the dataset id as a PARAMETER — they are Bright Data's wire, not
22
+ Instagram's — and re-implementing them here would be a second copy of the deferral handling, the
23
+ truncation guard, the SSRF rail and the snapshot-progress reader, i.e. five places for one bug.
24
+ Until wave 30 the code was right and the NAME was wrong: this file imported thirteen symbols from
25
+ `connectors_ig`, which read as a dependency on Instagram and was really a dependency on a supplier.
26
+ ⚠ **This file now imports ZERO names from `connectors_ig`, and a gate check asserts that**, because
27
+ the sentence above is the kind that quietly stops being true.
28
+
29
+ ⚠ **WHAT $0 COULD NOT BUY, so nobody reads this file as more measured than it is:**
30
+ 1. the real ROW shape — `/metadata` describes a DATASET, and Instagram's rows carry undeclared
31
+ envelope keys (`timestamp`, `input`) that no metadata call mentions;
32
+ 2. that a declared field POPULATES — Bright Data's Instagram Reels *declares* `views: number`
33
+ and delivers an account-grain wrong number (§4e). **Declared is not delivered**, and the one
34
+ TikTok claim that matters most (`play_count`) is exactly a declaration.
35
+ """
36
+ from __future__ import annotations
37
+
38
+ import automation_engine as engine
39
+ # ⚠ T09 — FOUR NAMES CAME OFF THIS LIST AND NOTHING BROKE, which is the point of deriving an
40
+ # import block from the AST both ways. `bd_call`, `bd_filter_start`, `bd_key` and `bd_ready` were
41
+ # imported here under a comment claiming they were *"re-exported for the runners"*; no runner ever
42
+ # read them off this module (the engine imports them from the transport itself), so they were four
43
+ # lines of dependency nobody was paying for. `[[artifact-with-no-importer]]` in its smallest form.
44
+ from connectors_bd import (
45
+ _bd_first_url,
46
+ _bd_flag,
47
+ _bd_list,
48
+ _bd_source_payload,
49
+ _first,
50
+ _ig_int,
51
+ bd_scrape,
52
+ )
53
+
54
+ # ---------------------------------------------------------------------------------------------
55
+ # THE DATASETS
56
+ # ---------------------------------------------------------------------------------------------
57
+ # ⚠ CATALOGUE PRESENCE IS NOT ENTITLEMENT — the same finding Instagram produced. `GET
58
+ # /datasets/list` returned 1,735 rows of which 12 are TikTok; the three below answer 200 with a
59
+ # full field list, and the two DISCOVERY halves answer **404 for our key**:
60
+ # `gd_lj71gn6l68bz7y9hc` (posts by profile) and `gd_lilwhto81z415d9mdl` (posts by keyword).
61
+ # ⇒ TikTok discovery routes through the PROFILES dataset's corpus filter, exactly as Instagram's
62
+ # does. A ticket that reaches for a by-keyword endpoint is reaching for a 404.
63
+ TT_DS_PROFILES = "gd_l1villgoiiidt09ci" # TikTok - Profiles. 40 fields, 152,000,000 records
64
+ TT_DS_POSTS = "gd_lu702nij2f790tmv9h" # TikTok - Posts. 43 fields
65
+ TT_DS_COMMENTS = "gd_lkf2st302ap89utw5k" # TikTok - Comments. 17 fields
66
+
67
+ #: The vendor's two post-type tokens, verbatim from the dataset's own `ai_description`
68
+ #: (*"strictly these two"*, video = 99.5% of rows). Ours are `image`/`video`/`carousel`.
69
+ TT_POST_TYPE_VIDEO = "video"
70
+ TT_POST_TYPE_CONTENT = "content"
71
+
72
+ #: What a TikTok profile URL looks like, for the runner that has a handle and needs a URL. Kept
73
+ #: beside the dataset ids because it is the same class of vendor fact.
74
+ #: ⚠ ONE SPELLING. `platform/core/user_tables.profile_url(handle, 'tiktok')` builds the identical
75
+ #: string from `_PROFILE_RULES` (contract C2, E's half) — this exists for the connector's own
76
+ #: batch calls, and the gate asserts the two agree rather than trusting that they do.
77
+ TT_PROFILE_URL = "https://www.tiktok.com/@{handle}"
78
+
79
+
80
+ def tt_profile_url(handle):
81
+ """`nurilab` → `https://www.tiktok.com/@nurilab`. `''` for a blank handle, never a bare `@`."""
82
+ h = str(handle or "").strip().lstrip("@")
83
+ return TT_PROFILE_URL.format(handle=h) if h else ""
84
+
85
+
86
+ # ---------------------------------------------------------------------------------------------
87
+ # THE FIELD MAPS
88
+ # ---------------------------------------------------------------------------------------------
89
+ # Each function turns ONE vendor row into the cell dict for one of the `ut_tt_*` schemas declared
90
+ # in `automation_engine`. The rules they all obey, stated once:
91
+ #
92
+ # * **BLANK MEANS NOT READ, NEVER "THEY HAVE NONE".** A key the vendor did not send is OMITTED,
93
+ # so a later, richer pull fills it instead of being overwritten by this one's silence. `_first`
94
+ # returns `None` (never 0) when nothing matches, which is what makes that possible.
95
+ # * **A zero from the vendor is a MEASUREMENT and survives.** (Instagram's `_ig_zero_is_blank`
96
+ # rule is scoped to a paid rung whose zeros were proven fictional; nothing here has earned it.)
97
+ # * **Every unpromoted vendor key stays whole in `source_payload`.** A schema addition on the
98
+ # vendor's side is preserved rather than silently discarded while our column model catches up.
99
+ # * **Nothing here writes a session token.** `tt_chain_token`, `secu_id` (~85% null), `short_id`
100
+ # (100% null in the sample), `ftc` (100% null) and `relation` are deliberately unmapped; they
101
+ # ride in `source_payload` where they make no claim.
102
+
103
+
104
+ def _tt_str(node, *names):
105
+ """The first non-empty string among `names`, or None when the vendor sent nothing.
106
+
107
+ ⛔ `None`, NOT `""`. The callers below drop `None` keys, which is what keeps a blank honest —
108
+ an empty string written into a cell claims "we looked and it is empty".
109
+ """
110
+ v = _first(node, *names)
111
+ if v is None:
112
+ return None
113
+ s = str(v).strip()
114
+ return s or None
115
+
116
+
117
+ def _tt_pct(node, *names):
118
+ """A vendor 0–1 engagement fraction → our stored 0–100 percentage, or None.
119
+
120
+ ⛔ THE ×100 IS NOT COSMETIC (wave 26, amendment C1-a). Our `pct` renderer appends the sign to
121
+ the STORED number, so writing the vendor's raw 0.0656 would print a 6.6% creator as `0.0%` —
122
+ measured on the Instagram side, and TikTok sends the same shape on all three of its rates.
123
+ """
124
+ v = _first(node, *names)
125
+ if v is None:
126
+ return None
127
+ out = engine._pct100(v)
128
+ return out or None
129
+
130
+
131
+ def _tt_day(node, *names):
132
+ """A vendor stamp → `YYYY-MM-DD`, or None. Our `date` columns store a day."""
133
+ v = _first(node, *names)
134
+ if v is None:
135
+ return None
136
+ return engine._day(v) or None
137
+
138
+
139
+ def _drop_blanks(row):
140
+ """The one place a mapped row loses its `None`s — see the BLANK MEANS NOT READ rule above."""
141
+ return {k: v for k, v in row.items() if v is not None and v != ""}
142
+
143
+
144
+ def normalize_profile(node, handle=""):
145
+ """A TikTok Profiles row → the `ut_tt_profile` / `ut_tt_snapshots` cell shape.
146
+
147
+ ⚠ TWO FIELDS READ THROUGH A CANDIDATE PAIR, and both pairs are the vendor's, not a guess:
148
+ * `biography` is PRIMARY and `signature` the FALLBACK — the probe measured `signature`
149
+ populated on 85% of rows and they carry the same text;
150
+ * `region` is PRIMARY and `country` the FALLBACK — `region` is the one with a documented
151
+ two-letter-ISO description, `country` has no description at all.
152
+ ⚠ `videos_count` is mapped to `posts_count` with a caveat recorded rather than hidden: its
153
+ `ai_description` ranges 1-89, so it may be a WINDOW rather than a lifetime total. It is the
154
+ only count of its kind the dataset offers.
155
+ """
156
+ node = node if isinstance(node, dict) else {}
157
+ account = _tt_str(node, "account_id") or str(handle or "").strip().lstrip("@")
158
+ return _drop_blanks({
159
+ "platform": engine.PLATFORM_TIKTOK,
160
+ "handle": account,
161
+ "full_name": _tt_str(node, "nickname"),
162
+ "tt_id": _tt_str(node, "id"),
163
+ "profile_url": _tt_str(node, "url") or (tt_profile_url(account) or None),
164
+ "bio": _tt_str(node, "biography", "signature"),
165
+ "external_url": _bd_first_url(_first(node, "bio_link")),
166
+ "verified": _bd_flag(node, "is_verified"),
167
+ "is_private": _bd_flag(node, "is_private"),
168
+ # ⚠ APPROXIMATE, and the word is the vendor's: `is_commerce_user` has *"many null values"*
169
+ # on their own description. It is the closest thing TikTok has to Instagram's
170
+ # `is_business_account`, and `_bd_flag` writes nothing at all when the key is absent — so
171
+ # the approximation only ever fills a cell the vendor actually answered.
172
+ "is_business": _bd_flag(node, "is_commerce_user"),
173
+ "followers": _ig_int(_first(node, "followers")),
174
+ "following": _ig_int(_first(node, "following")),
175
+ "posts_count": _ig_int(_first(node, "videos_count")),
176
+ # ⚠ `likes` on a PROFILE row is likes RECEIVED across the account's videos (18-110,200, no
177
+ # nulls). It is not a post-level number and it is not our `likes` column, which is why it
178
+ # is stored under a different name.
179
+ "likes_received": _ig_int(_first(node, "likes")),
180
+ "avg_engagement": _tt_pct(node, "awg_engagement_rate"),
181
+ "like_engagement": _tt_pct(node, "like_engagement_rate"),
182
+ "comment_engagement": _tt_pct(node, "comment_engagement_rate"),
183
+ "country_code": _tt_str(node, "region", "country"),
184
+ "region": _tt_str(node, "region"),
185
+ "predicted_lang": _tt_str(node, "predicted_lang"),
186
+ # ⚠ ACCOUNT AGE, NOT A MEASUREMENT STAMP — `create_time` on a profile is when the ACCOUNT
187
+ # was made. TikTok stamps nothing with "when this number was true", exactly like Instagram,
188
+ # which is why the append law dates a snapshot by when WE read it.
189
+ "account_created_at": _tt_day(node, "create_time"),
190
+ "source_payload": _bd_source_payload(node),
191
+ })
192
+
193
+
194
+ def tt_post_type(node):
195
+ """A TikTok post row → one of OUR three type options, or None.
196
+
197
+ ⛔ THE FIRST OF THE PROBE DOC'S TWO NAMED BLOCKERS. The vendor's vocabulary is `"video"` /
198
+ `"content"`; ours is `image` / `video` / `carousel` and has no `"content"`. Writing the
199
+ vendor's token would fail `_clean_field`'s option check on the way in and would put an
200
+ untranslated API word in front of a user on the way out.
201
+
202
+ So: `video` is `video`, and `content` — TikTok's photo-mode post — is `image`, EXCEPT when the
203
+ row carries more than one `carousel_images` entry, which is what a carousel IS on either
204
+ network. ⚠ The multi-image branch is decided from the IMAGES, never from the type token: the
205
+ token cannot express it, so inferring `carousel` from the word would be inventing a fact.
206
+ ⚠ An UNKNOWN token returns None rather than defaulting to `video` (99.5% of rows are video, and
207
+ that is exactly what would make the wrong default invisible).
208
+ """
209
+ raw = str((node or {}).get("post_type") or "").strip().lower()
210
+ if raw == TT_POST_TYPE_VIDEO:
211
+ return "video"
212
+ if raw != TT_POST_TYPE_CONTENT:
213
+ return None
214
+ images = (node or {}).get("carousel_images")
215
+ return "carousel" if isinstance(images, list) and len(images) > 1 else "image"
216
+
217
+
218
+ def normalize_post(node):
219
+ """A TikTok Posts row → the `ut_tt_posts` cell shape. None when it carries no identity.
220
+
221
+ ⛔ THE SECOND NAMED BLOCKER, RESOLVED HERE AND NOWHERE ELSE: `play_count` is ONE number and
222
+ Instagram's schema has TWO columns for it (`plays` and `views`). On TikTok they are the same
223
+ fact — `play_count` IS the count TikTok displays under a video — so it maps to `views` and
224
+ `ut_tt_posts` HAS NO `plays` COLUMN. Copying one vendor number into two of our columns would
225
+ manufacture a second measurement that a rollup could average or double-count, which is a worse
226
+ outcome than the missing column it would paper over.
227
+
228
+ ⚠ `shortcode` reads `shortcode` then `post_id`: both are 19-digit numerics on this dataset and
229
+ the probe measured them as the same shape. That equality is what lets the comments→posts link
230
+ join with NO normaliser, which the Instagram side never had.
231
+ ⚠ `num_share_count` (a number) is preferred over `share_count` (typed TEXT by the vendor).
232
+ ⚠ `commerce_info` is a business/commerce LOCATION per its own description — cities and
233
+ countries. It is NOT a paid-partnership flag, and nothing in this dataset is: TikTok declares
234
+ no equivalent, so `paid_partnership`/`partner` have no column on this family at all.
235
+ """
236
+ node = node if isinstance(node, dict) else {}
237
+ shortcode = _tt_str(node, "shortcode", "post_id")
238
+ if not shortcode:
239
+ return None
240
+ return _drop_blanks({
241
+ "platform": engine.PLATFORM_TIKTOK,
242
+ "shortcode": shortcode,
243
+ # ⛔⛔ `account_id`, NOT `profile_username` — MEASURED on a real Posts row 2026-08-12.
244
+ # The vendor's `profile_username` is the DISPLAY NAME (`"Dina"`), while `account_id` is the
245
+ # @handle (`"d1na_th"`) — the same field `normalize_profile` already reads for `handle`, so
246
+ # one name means one thing across both corpora. Reading the display name silently broke the
247
+ # only join this table has: `ut_tt_posts.influencer_key` -> `ut_tt_profile.handle` matched
248
+ # NOTHING, so a person could not filter posts by creator and a rollup would count zero.
249
+ # ⚠ NO FALLBACK TO `profile_username`, deliberately. It is not a degraded handle, it is a
250
+ # different fact, and filling a join key with it is worse than leaving it blank — a blank
251
+ # is visibly missing, a display name looks like an answer [[one-question-two-normalizers]].
252
+ # The URL is the honest second source: it carries the handle by construction.
253
+ "influencer_key": (_tt_str(node, "account_id")
254
+ or tt_handle(_tt_str(node, "profile_url", "url") or "") or None),
255
+ "posted_at": _tt_day(node, "create_time"),
256
+ "type": tt_post_type(node),
257
+ "caption": _tt_str(node, "description"),
258
+ "url": _tt_str(node, "url"),
259
+ "hashtags": _bd_list(node, "hashtags"),
260
+ "tagged_location": _tt_str(node, "commerce_info"),
261
+ "views": _ig_int(_first(node, "play_count")),
262
+ "likes": _ig_int(_first(node, "digg_count")),
263
+ "comments": _ig_int(_first(node, "comment_count")),
264
+ "shares": _ig_int(_first(node, "num_share_count")),
265
+ "saves": _ig_int(_first(node, "collect_count")),
266
+ "video_duration": _ig_int(_first(node, "video_duration")),
267
+ "source_payload": _bd_source_payload(node),
268
+ })
269
+
270
+
271
+ def normalize_comment(node):
272
+ """A TikTok Comments row → the `ut_tt_comments` cell shape. None without a comment id.
273
+
274
+ ⚠ THE NAME COLLISION, RESOLVED: TikTok's `replies` is an ARRAY of reply objects and OUR
275
+ `replies` column is an INT count. The count comes from `num_replies`; the array stays whole in
276
+ `source_payload`. Reading the array's length instead would be a second, disagreeing answer to
277
+ a question the vendor already answers — and it would disagree, because a page of replies is not
278
+ all of them.
279
+ ⚠ `date_created` is typed `date` by this vendor, unlike Instagram's `comment_date` which is
280
+ text and needs defensive parsing. It still goes through `_tt_day` — one date path, so a vendor
281
+ that changes its mind cannot change ours.
282
+ ⚠ The comment TEXT and every identifiable commenter field (`commenter_user_name` is flagged
283
+ PII) stay in `source_payload` and are promoted to no column, which is the same posture the
284
+ Instagram comment schema takes for the same D-24 reason.
285
+ """
286
+ node = node if isinstance(node, dict) else {}
287
+ comment_key = _tt_str(node, "comment_id")
288
+ if not comment_key:
289
+ return None
290
+ return _drop_blanks({
291
+ "platform": engine.PLATFORM_TIKTOK,
292
+ "comment_key": comment_key,
293
+ "shortcode": _tt_str(node, "post_id"),
294
+ # ⭐⭐ OWNER RULING 2026-08-12: the comment's CONTENT gets a column. `comment_text` is the
295
+ # vendor's own key and `comment_text_only` its stripped variant (the probe recorded both);
296
+ # primary first, so a row carrying the rich form is not silently served the plain one.
297
+ # ⛔ The commenter's identity is deliberately NOT promoted — see `TT_COMMENT_FIELDS`.
298
+ "text": _tt_str(node, "comment_text", "comment_text_only"),
299
+ "commented_at": _tt_day(node, "date_created"),
300
+ "likes": _ig_int(_first(node, "num_likes")),
301
+ "replies": _ig_int(_first(node, "num_replies")),
302
+ "source_payload": _bd_source_payload(node),
303
+ })
304
+
305
+
306
+ #: ⭐ The three maps, addressable by name — so a gate (and the runners in T04-T06) can walk them
307
+ #: rather than naming three functions, and so adding a fourth dataset is one entry.
308
+ TT_NORMALIZERS = {
309
+ "tt_profile": normalize_profile,
310
+ "tt_post_metrics": normalize_post,
311
+ "tt_comments": normalize_comment,
312
+ }
313
+
314
+
315
+ # ---------------------------------------------------------------------------------------------
316
+ # THE FETCH — wave 30 · W30-T08 (carrying wave-29's dropped T05)
317
+ # ---------------------------------------------------------------------------------------------
318
+
319
+ def tt_handle(url):
320
+ """A TikTok profile URL **or** a bare handle → the handle. `''` when it is neither.
321
+
322
+ Deliberately permissive about the input and strict about the output, because the two callers
323
+ hand it different things: an automation stores whatever a person typed in the profile column
324
+ (`@nurilab`, `nurilab`, or the full URL), while the discovery runner already holds a clean
325
+ `account_id`. One normaliser, so a row found by discovery and a row typed by hand cannot
326
+ resolve to two different handles.
327
+ """
328
+ s = str(url or "").strip()
329
+ if not s:
330
+ return ""
331
+ if "tiktok.com" in s.lower():
332
+ # Everything after the first `@`, up to the next path segment or query.
333
+ tail = s.split("@", 1)[1] if "@" in s else ""
334
+ s = tail.split("/")[0].split("?")[0].split("#")[0]
335
+ s = s.strip().lstrip("@").strip()
336
+ # A handle is the vendor's `account_id` shape: alphanumerics, dots and underscores.
337
+ return s if s and all(c.isalnum() or c in "._" for c in s) else ""
338
+
339
+
340
+ def tt_post_urls(node, limit=0):
341
+ """⭐ WAVE 30 · T10 — the profile row's own post permalinks, newest-first as the vendor sends.
342
+
343
+ ⛔ THIS IS WHY TIKTOK POST CAPTURE COSTS NO EXTRA DISCOVERY. `top_videos` rides the PROFILE row
344
+ we have already bought, so the posts read is a scrape of links we hold, never a search for them.
345
+ The two TikTok DISCOVERY datasets (posts-by-profile, posts-by-keyword) are **404 for our key**,
346
+ so a design that reached for either would not merely be dearer, it would not work.
347
+
348
+ ⛔⛔ CORRECTED 2026-08-12 — THIS DOCSTRING USED TO SAY *"the probe MEASURED `top_videos` as an
349
+ array of video permalinks, NO empties"*, AND THAT SENTENCE IS WHAT SHIPPED THE BUG. The probe
350
+ read `/datasets/{id}/metadata` — a DATASET description — and the phrase quoted was the field's
351
+ `ai_description`, not an observation of a row. The real row sends **dicts keyed `video_url`**
352
+ (measured below), so the reader built against the quoted sentence found nothing, forever, in
353
+ silence. ⭐ The transferable half: *"the probe measured X"* and *"the probe read a declaration
354
+ of X"* are different claims, and prose cannot be told apart by a reader downstream — which is
355
+ why the correction names the method, not just the value.
356
+
357
+ ⚠ `top_posts_data` is deliberately NOT read: the probe calls it *"a thin dup of `top_videos`"*,
358
+ and preferring whichever happened to be longer is how one creator's window silently differs
359
+ from another's.
360
+ ⚠ `limit <= 0` means "everything the row carried". The CAP IS THE CALLER'S — `config.maxPosts`,
361
+ validated 1..12 — and it is applied here rather than after the scrape so an unwanted post is
362
+ never bought. [[a-constant-two-features-share]]: the 12 is the vendor's measured profile window,
363
+ not a number this function may invent.
364
+ """
365
+ raw = (node or {}).get("top_videos")
366
+ out = []
367
+ for item in raw if isinstance(raw, list) else []:
368
+ # ⛔⛔ MEASURED ON A REAL ROW 2026-08-12, AND IT IS NOT WHAT THE SCHEMA SAID.
369
+ # `top_videos` is NOT an array of permalink strings. One paid TikTok Profiles scrape of a
370
+ # live handle returns **19 DICTS**, keyed
371
+ # `video_url · video_id · playcount · diggcount · commentcount · share_count ·
372
+ # favorites_count · create_date · cover_image`.
373
+ # The docstring above cites the $0 probe as having "measured" permalinks — it had not, and
374
+ # could not: `/datasets/{id}/metadata` describes a DATASET, and the probe's own verdict says
375
+ # so in terms (*"what $0 cannot buy … the real ROW shape … that a declared field
376
+ # POPULATES"*, `wave29/proto/tiktok-schema.md`). This is the SECOND time this vendor's
377
+ # declaration has diverged from its delivery on this exact axis; BD's IG Reels `views` was
378
+ # the first. [[reachable-is-not-the-same-as-built]]
379
+ # ⇒ The consequence, live: `TT_DS_POSTS` was never reached, because this returned an EMPTY
380
+ # list on every real profile — post capture could not have worked for anybody, and the
381
+ # T10 gate stayed green because its canned fixture encoded the DECLARED shape. A fixture
382
+ # written from a schema tests the schema.
383
+ # ⚠ `video_url` is FIRST because it is the key the vendor actually sends; `url` is kept
384
+ # because it costs nothing and is what a future corpus revision would most likely use. The
385
+ # bare-string branch stays for the same reason — this widens what is ACCEPTED and invents
386
+ # nothing: a shape that yields no `http…` value still degrades to "no posts", exactly as
387
+ # before, rather than to a URL built out of a guess.
388
+ # ⚠ `top_posts_data` is STILL not read (it carries `post_url` and would work): preferring
389
+ # whichever array happened to be longer is how one creator's window silently differs from
390
+ # another's, and that reasoning is unchanged by this correction.
391
+ if isinstance(item, dict):
392
+ u = str(item.get("video_url") or item.get("url") or "").strip()
393
+ else:
394
+ u = str(item or "").strip()
395
+ if u.lower().startswith("http") and u not in out:
396
+ out.append(u)
397
+ return out[:limit] if limit and limit > 0 else out
398
+
399
+
400
+ def pull_posts_tt(post_urls, log=print, deferred=None):
401
+ """The TikTok Posts dataset for a list of permalinks → `(rows, note)`, already normalised.
402
+
403
+ ⚠ ONE CALL FOR THE WHOLE WINDOW. `bd_scrape` has always taken a list, and the Instagram side
404
+ measured what happens when a caller forgets: 25 records, one billed snapshot each, a walk still
405
+ running at 67 minutes. Nothing here loops per URL.
406
+ """
407
+ urls = [str(u) for u in (post_urls or []) if str(u or "").strip()]
408
+ if not urls:
409
+ return [], ""
410
+ rows, note = bd_scrape(TT_DS_POSTS, urls, deferred=deferred)
411
+ if note:
412
+ log(f"[aios-tt] posts: {note}")
413
+ return [], note
414
+ out = [r for r in (normalize_post(n) for n in rows) if r]
415
+ return out, ""
416
+
417
+
418
+ def pull_comments_tt(post_urls, log=print, deferred=None):
419
+ """The TikTok Comments dataset for a list of POST permalinks → `(rows, note)`, normalised.
420
+
421
+ ⛔ THE MOST EXPENSIVE THING THIS PRODUCT BUYS, and the reason `commentMetrics` defaults OFF on
422
+ both networks: a comments scrape ingests identifiable third parties who never entered anybody's
423
+ list (D-24). The mapper already keeps every commenter field in `source_payload` and promotes
424
+ none of them to a column; this function adds no new exposure, it just has to be asked for.
425
+ """
426
+ urls = [str(u) for u in (post_urls or []) if str(u or "").strip()]
427
+ if not urls:
428
+ return [], ""
429
+ rows, note = bd_scrape(TT_DS_COMMENTS, urls, deferred=deferred)
430
+ if note:
431
+ log(f"[aios-tt] comments: {note}")
432
+ return [], note
433
+ out = [r for r in (normalize_comment(n) for n in rows) if r]
434
+ return out, ""
435
+
436
+
437
+ #: ⭐⭐ WAVE 30 · D-156 — THE MEDIA DATASETS, AS A SET, SO THE HAND-OFF CAN FILTER ON IDENTITY.
438
+ #: `_media_deferrals` uses this to lift ONLY posts/comments snapshots out of the local deferral
439
+ #: list. That is what makes it structurally impossible to file a PROFILE snapshot in the engine's
440
+ #: metric queue — the defect a draft of T10 shipped and A-39 booked as "the wrong fix is worse
441
+ #: than the gap". A membership test cannot be got wrong by a later edit the way `if` order can.
442
+ TT_MEDIA_DATASETS = (TT_DS_POSTS, TT_DS_COMMENTS)
443
+
444
+
445
+ def _media_deferrals(deferred):
446
+ """The POSTS/COMMENTS entries of a `bd_scrape` deferral list — never the profile's.
447
+
448
+ ⚠ The engine, not this module, decides what a deferral MEANS: it stamps `kind` and the
449
+ handle and files it. TikTok needs no `_tag_metric_deferrals` twin because one dataset is one
450
+ kind here, so the id already carries everything a mapper choice depends on — and importing
451
+ Instagram's tagger is not available anyway (W30-T09 gates ZERO `from connectors_ig` lines).
452
+ """
453
+ out = []
454
+ for d in deferred or []:
455
+ if isinstance(d, dict) and str(d.get("datasetId") or "") in TT_MEDIA_DATASETS:
456
+ out.append(dict(d))
457
+ return out
458
+
459
+
460
+ def pull_profile_tt(url, log=print, pending_profile=None, prefetch=None,
461
+ max_posts=0, post_metrics=False, comment_metrics=False):
462
+ """ONE TikTok profile from the vendor. Same return contract as `pull_profile`.
463
+
464
+ `{state, profile, posts, comments, via, note}` with `state ∈ ok | partial | blocked | error`,
465
+ so the engine's enrich branch treats every network identically and no caller learns a new
466
+ shape.
467
+
468
+ ⭐ WAVE 30 · T10 — POSTS AND COMMENTS ARE REAL NOW, AND BOTH DEFAULT OFF, exactly as Instagram's
469
+ do. `post_metrics` scrapes the profile row's own `top_videos` permalinks (see `tt_post_urls` —
470
+ no discovery call, because both TikTok discovery datasets 404 for our key); `comment_metrics`
471
+ then scrapes the comments of the posts that came back. ⚠ COMMENTS REQUIRE POSTS by construction
472
+ rather than by a rule: their input IS a post permalink, so asking for comments with post capture
473
+ off is a request with no subject, and it returns none instead of quietly buying posts nobody
474
+ asked for.
475
+
476
+ ⛔ `partial` IS THE SUCCESS STATE WHENEVER NO MEDIA WAS READ, and that is deliberate rather than
477
+ pessimistic. The Instagram contract reads `ok` only when identity AND media both landed
478
+ (`pull_profile_bd`: *"identity without media is still partial ... a run that wrote a follower
479
+ count and no posts must not paint green over a posts table that did not grow"*). So: posts not
480
+ ASKED for → `partial`, saying so; posts asked for and landed → `ok`; asked for and none came →
481
+ `partial` with the vendor's reason. The state answers "did this pull deliver what it went for",
482
+ never "did the function finish".
483
+
484
+ ⚠ **NO FREE RUNG, AND NO FALLBACK CHAIN.** Instagram's `pull_profile` drops to Apify when the
485
+ paid rung refuses; `providers.DEFAULT_CHAINS["tt_profile"]` is deliberately single-provider,
486
+ with its own note explaining that a multi-provider chain is a promise something walks it and
487
+ that nothing walks Instagram's second name today either. So a refusal here is final, and it
488
+ says so instead of implying a retry somewhere.
489
+ """
490
+ handle = tt_handle(url)
491
+ if not handle:
492
+ return {"state": "error", "profile": {}, "posts": [], "comments": [], "via": "",
493
+ "note": f"{url!r} is not a TikTok profile URL or handle"}
494
+
495
+ # ⭐ THE BATCH FAST PATH, same shape as the Instagram side: `prefetch` is `{handle: node}` from
496
+ # one multi-URL scrape covering a whole selection. A hit is a vendor round trip that does not
497
+ # happen; a miss falls through to the single-URL call below.
498
+ cached = prefetch.get(handle) if isinstance(prefetch, dict) else None
499
+ _deferred = []
500
+ if isinstance(cached, dict) and cached:
501
+ rows, note = [cached], ""
502
+ else:
503
+ rows, note = bd_scrape(TT_DS_PROFILES, [tt_profile_url(handle)], deferred=_deferred)
504
+
505
+ node = rows[0] if rows else {}
506
+ profile = normalize_profile(node, handle) if node else {}
507
+ # ⛔ THE READABILITY TEST IS `followers`/`following`, NOT "did we get a dict". `normalize_profile`
508
+ # drops blanks, so an unreadable row still returns `{"platform": …, "handle": …}` — truthy, and
509
+ # carrying nothing anybody asked for. The Instagram rung tests exactly this pair for exactly
510
+ # this reason, and answering "0 followers" instead is the failure it exists to prevent.
511
+ unreadable = profile.get("followers") is None and profile.get("following") is None
512
+ if note or unreadable:
513
+ # ⭐ THE DEFERRAL IS HANDED OVER RATHER THAN DISCARDED. A snapshot the vendor is still
514
+ # building HAS ALREADY BEEN PAID FOR; dropping its id bills again on the next run for the
515
+ # same record. That was live on the Instagram profile path until 2026-08-09 — measured on
516
+ # nurilab as two runs, two fresh snapshots, both abandoned — and it is not being
517
+ # reintroduced here by omission.
518
+ if isinstance(pending_profile, list):
519
+ for d in _deferred:
520
+ pending_profile.append({**d, "kind": "profile", "influencer": handle})
521
+ why = note or ("the scrape answered, but no follower/following counts were readable in it "
522
+ "(the field names may have moved - see tiktok-capture.md)")
523
+ return {"state": "blocked", "profile": {}, "posts": [], "comments": [], "via": "brightdata",
524
+ "deferredProfile": [d.get("snapshotId") for d in _deferred],
525
+ "note": why}
526
+
527
+ # --- W30-T10: THE MEDIA, ONLY WHEN IT WAS ASKED FOR. ------------------------------------
528
+ if not post_metrics:
529
+ return {"state": "partial", "profile": profile, "posts": [], "comments": [],
530
+ "via": "brightdata",
531
+ "note": note or "profile read; post capture is off for this step"}
532
+ urls = tt_post_urls(node, limit=max_posts)
533
+ if not urls:
534
+ # ⚠ NOT AN ERROR AND NOT A RETRY. A creator with no `top_videos` has nothing to buy, and
535
+ # saying so is what stops the next run paying to be told the same thing.
536
+ return {"state": "partial", "profile": profile, "posts": [], "comments": [],
537
+ "via": "brightdata",
538
+ "note": note or "profile read; this account's row carried no post links"}
539
+ posts, p_note = pull_posts_tt(urls, log=log, deferred=_deferred)
540
+ comments, c_note = ([], "")
541
+ if comment_metrics and posts:
542
+ # The comments dataset is keyed on a POST permalink, so it reads the posts we just bought —
543
+ # `url` from the mapper, never the profile's raw array, so a post the posts scrape refused
544
+ # is not silently asked about again one rung later.
545
+ comments, c_note = pull_comments_tt([p.get("url") for p in posts if p.get("url")],
546
+ log=log, deferred=_deferred)
547
+ # ⭐⭐ WAVE 30 · D-156 — THE MEDIA DEFERRALS ARE HANDED BACK, and the shape of the hand-off is
548
+ # the whole lesson. An earlier draft of T10 appended every `_deferred` entry to
549
+ # `pending_profile` tagged `kind: "profile"`. By the time control reaches here a PROFILE
550
+ # deferral is impossible — the profile branch above returns `blocked` on any note — so **every
551
+ # id fanned out that way was a POSTS or COMMENTS snapshot in the PROFILE queue**, whose
552
+ # collector writes preset profile cells onto somebody's record from post rows. The engine keeps
553
+ # the two queues apart deliberately (`_pending_profile_tasks` vs `_pending_metric_tasks`).
554
+ # ⇒ So this returns them under their OWN key, filtered by dataset identity
555
+ # (`_media_deferrals`), and the engine files them in the metric queue with the handle it
556
+ # already holds. Returning rather than appending also keeps the queue's vocabulary out of a
557
+ # connector: this module knows which CORPUS deferred, never what the engine calls it.
558
+ # ⚠ `deferredMedia` rides BOTH returns on purpose. The empty-posts case is the one that
559
+ # matters most — that is exactly the run where the vendor took too long, so a caller reading
560
+ # the ids only from the success path would lose every batch it actually paid for.
561
+ deferred_media = _media_deferrals(_deferred)
562
+ if not posts:
563
+ return {"state": "partial", "profile": profile, "posts": [], "comments": [],
564
+ "via": "brightdata", "deferredMedia": deferred_media,
565
+ "note": p_note or note or "profile read; the post source returned nothing"}
566
+ return {"state": "ok", "profile": profile, "posts": posts, "comments": comments,
567
+ "via": "brightdata", "deferredMedia": deferred_media,
568
+ "note": c_note or note or ""}
api/main.py CHANGED
The diff for this file is too large to render. See raw diff
 
api/odoo_relational.py CHANGED
The diff for this file is too large to render. See raw diff
 
api/providers.py CHANGED
The diff for this file is too large to render. See raw diff
 
api/routes_admin.py CHANGED
The diff for this file is too large to render. See raw diff
 
api/routes_agent_harness.py CHANGED
@@ -1,455 +1,455 @@
1
- """routes_agent_harness.py — CONTRACT C4: the agent's HARNESS, kept as versioned files.
2
-
3
- Owner item 3, verbatim (2026-08-18): *"This new module under agent is supposed to host any file
4
- pertaining to the agent skills, router etc. So we build a user can build a custom harness for each
5
- agent through the chat interface."*
6
-
7
- GET /api/v1/agents/{id}/harness the file LIST (no bodies)
8
- GET /api/v1/agents/{id}/harness?path=… one file: current body + its versions
9
- GET /api/v1/agents/{id}/harness?path=…&version=N one older body, verbatim
10
- PUT /api/v1/agents/{id}/harness write a NEW version of one file
11
- DELETE /api/v1/agents/{id}/harness?path=… drop a file and its history
12
-
13
- ⭐⭐ **R8 IS THE WHOLE SHAPE: BOTH PRINCIPALS WRITE, AND NOTHING IS EVER OVERWRITTEN.** A write
14
- appends a version; a roll-back is a write of an older body (`restoredFrom`), never a delete. So the
15
- history is a record of what happened rather than of what somebody last wanted it to look like.
16
-
17
- ⛔ **R8 IS DELIBERATELY NOT R9.** An agent-authored ACTION's configuration is agent-only (item 8,
18
- `routes_automation`); a harness file is not. Do not copy this file's posture over there or that
19
- one's over here — the two rulings differ on purpose and they sit one screen apart in the product.
20
-
21
- ⚠ **`author` IS THE SESSION, `authorKind` IS THE PROVENANCE, AND THEY ARE DIFFERENT FACTS.** Every
22
- write through this router is made BY a signed-in administrator, so `author` is stamped from the
23
- session and can never be supplied by the caller. `authorKind` says whether the BODY was drafted by
24
- a person or by the agent in the chat — a claim the client is entitled to make, because both are
25
- permitted (R8) and so nothing is bought by forging it. `record_version()` below is the server-side
26
- door the automation engine uses when the agent writes with no session at all; that one stamps the
27
- agent's own id as the author, which is the only case where `author` is not a username.
28
-
29
- ⛔ **THE VERSION LIST IS CAPPED AND THE CAP IS REPORTED, NEVER SILENT.** `MAX_VERSIONS` versions of
30
- one file are kept; past that the OLDEST are dropped and the count of what was dropped rides in
31
- `trimmed` on every payload that mentions the file, so a reader can see that the history is partial
32
- rather than infer that the file was only ever saved twice. This is the tenant document, which is
33
- already 28.6 MB on tenant #0 and is deep-copied on every read: an unbounded per-agent history is
34
- a store-sized leak with a UI in front of it.
35
- """
36
- from datetime import datetime, timezone
37
-
38
- from fastapi import APIRouter, Body, Depends
39
-
40
- from deps import Session, err, require_session
41
-
42
- router = APIRouter(prefix="/api/v1")
43
-
44
- #: The tenant's harness files: `{agent_id: {path: file_record}}`. A per-tenant bucket, so it rides
45
- #: `runtime.store_key`'s prefix and never lands in tenant #0's namespace — the same rule
46
- #: `routes_slack.AGENTS_KEY` follows for the agent records these hang off.
47
- HARNESS_KEY = "agent_harness"
48
-
49
- #: One body. Generous for a skill or a router file and far under the point where a single write
50
- #: would move the tenant document measurably. A larger body is a REFUSAL at the door, not a
51
- #: truncation: a silently truncated skill file is a harness that does not do what its text says.
52
- MAX_BODY_BYTES = 128 * 1024
53
-
54
- #: Files per agent. A refusal, not a trim — creating the 65th file is a different act from saving
55
- #: the 101st version of one, and only the second can be a routine consequence of ordinary editing.
56
- MAX_FILES = 64
57
-
58
- #: Versions kept per file. Past this the oldest go and `trimmed` counts them (see the header).
59
- MAX_VERSIONS = 100
60
-
61
- MAX_PATH = 200
62
-
63
- #: What a path may contain. ⛔ THIS IS NOT A FILESYSTEM PATH AND NOTHING HERE EVER TOUCHES A DISK —
64
- #: the "files" are keys in a store bucket. The character rule exists so the key is displayable, is
65
- #: safe to put in a URL, and cannot carry a traversal sequence that would look meaningful to a
66
- #: future reader who assumes it IS a filesystem path. Fail-closed on the character set, not on a
67
- #: list of forbidden sequences: an allow-list cannot be walked around by a spelling.
68
- _PATH_OK = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-/")
69
-
70
- AUTHOR_KINDS = ("user", "agent")
71
-
72
-
73
- def _now():
74
- return datetime.now(timezone.utc).isoformat(timespec="seconds")
75
-
76
-
77
- def _all(rt):
78
- """`{agent_id: {path: record}}` for one tenant. `{}` on any failure — an unreadable bucket must
79
- degrade to "this agent has no harness files", never to a 500 on the pane that lists them."""
80
- try:
81
- found = rt.get(HARNESS_KEY) or {}
82
- except Exception: # noqa: BLE001
83
- return {}
84
- return found if isinstance(found, dict) else {}
85
-
86
-
87
- def _files(rt, agent_id):
88
- found = _all(rt).get(str(agent_id))
89
- return found if isinstance(found, dict) else {}
90
-
91
-
92
- #: What kind of principal an id names. `None` = this tenant has no agent with that id.
93
- #:
94
- #: ⛔⛔ THERE ARE TWO AGENT REGISTRIES IN THIS PRODUCT AND THE FIRST DRAFT OF THIS FILE KNEW ONLY
95
- #: ONE (ASK D-5, 2026-08-18). `routes_slack._agents` is the per-Slack-channel permission wall under
96
- #: Manage users. The **Agents module** the owner's item 3 is about is the AUTOMATION surface —
97
- #: `Shell.tsx` mounts `<Lazily surface="Agents"><AutomationSurface /></Lazily>`, and
98
- #: `AutomationDetail`'s `panelTabs` is literally the `Properties | Run history` strip R7 adds
99
- #: "Harness" to. Keyed on the Slack bucket alone, every `GET/PUT /agents/{id}/harness` from that
100
- #: tab would have answered **404 no_agent**: whole, gate-green and dead on arrival
101
- #: [[reachable-is-not-the-same-as-built]]. Verified independently before acting, not taken on
102
- #: report: `surface="Agents"` mounts `AutomationSurface`, and `ManageAgentPane` contains ZERO
103
- #: occurrences of Canvas, Properties, Run history or panelTabs.
104
- #:
105
- #: ⭐ ONE STORE, BOTH PRINCIPALS — never a second bucket keyed by surface. The owner's "agent
106
- #: skills, router" files in two places is the parallel code path item 13 exists to refuse.
107
- AGENT_SLACK = "slack"
108
- AGENT_AUTOMATION = "automation"
109
-
110
-
111
- def agent_kind(rt, agent_id):
112
- """Which registry holds this id — `AGENT_SLACK`, `AGENT_AUTOMATION`, or `None`.
113
-
114
- ⚠ A SYNTHETIC ROW IS NOT AN AGENT FOR THIS PURPOSE. `field:` and `system:` ids are DERIVED at
115
- read time from a column definition or a connector schedule; they have no stored home, so a
116
- harness file hung off one is an orphan the moment the column changes. `patch_automation`
117
- already refuses those ids for the neighbouring reason, and this refuses them by simply not
118
- finding them — `all_definitions` holds stored automations only.
119
- """
120
- import routes_slack
121
- aid = str(agent_id or "")
122
- if isinstance(routes_slack._agents(rt).get(aid), dict):
123
- return AGENT_SLACK
124
- import automation_engine as engine
125
- try:
126
- known = engine.all_definitions(rt) or {}
127
- except Exception: # noqa: BLE001
128
- return None
129
- return AGENT_AUTOMATION if aid in known else None
130
-
131
-
132
- def _known_agent(rt, agent_id):
133
- """Does this tenant have an agent with this id, in EITHER registry?"""
134
- return agent_kind(rt, agent_id) is not None
135
-
136
-
137
- def agent_wall(session, agent_id):
138
- """404 for an unknown id, else apply the wall THAT PRINCIPAL'S OWN SURFACE applies.
139
-
140
- ⛔⛔ ONE DOOR, TWO WALLS, AND THAT IS NOT A SECOND CODE PATH — it is the refusal to invent a
141
- THIRD wall. A Slack channel agent is administered under Manage users and every `/agents/*` door
142
- in `routes_slack` is `admin_gate`; an automation lives in the Agents module and every door in
143
- `routes_automation` is `module_gate("automation")`. A harness file is configuration OF the
144
- agent it hangs off, so it is reached by whoever may already configure that agent. Picking one
145
- of the two walls for both would either lock the Agents module's own users out of a tab the
146
- owner asked for, or hand the Slack permission wall to anyone with an automation grant.
147
- """
148
- kind = agent_kind(session.runtime, agent_id)
149
- if kind is None:
150
- raise err(404, "no_agent", "there is no agent with that id in this workspace")
151
- if kind == AGENT_SLACK:
152
- import core.perms as perms
153
- if not perms.is_admin(session.user):
154
- raise err(403, "forbidden", "administrators only")
155
- else:
156
- session.require("automation")
157
- return kind
158
-
159
-
160
- def normalize_path(raw):
161
- """THE path rule, in ONE place. Returns the cleaned path, or `None` if it is not acceptable.
162
-
163
- ⛔⛔ ONE RULE, TWO DOORS, AND THAT IS WHY THIS IS A FUNCTION RATHER THAN TWO IF-BLOCKS. There
164
- are two ways into this store — the HTTP route (which must answer 400) and `record_version()`
165
- (which must raise `ValueError`, having no response to put a status into). Written twice, the
166
- two copies are [[one-question-two-normalizers]] waiting to happen: the first weakening of one
167
- copy is invisible because the other still refuses, so nothing goes red and the wall is now
168
- half there. Written once, a change to the rule is felt at both doors and by the gate.
169
- """
170
- path = str(raw or "").strip().strip("/")
171
- if not path or len(path) > MAX_PATH:
172
- return None
173
- if set(path) - _PATH_OK or ".." in path or "//" in path:
174
- return None
175
- return path
176
-
177
-
178
- def _clean_path(raw):
179
- """`normalize_path` at the HTTP door, where a refusal is a 400 with a reason."""
180
- path = normalize_path(raw)
181
- if path is None:
182
- if not str(raw or "").strip().strip("/"):
183
- raise err(400, "no_path", "a harness file needs a path, for example skills/router.md")
184
- if len(str(raw)) > MAX_PATH:
185
- raise err(400, "path_too_long", f"a harness path is at most {MAX_PATH} characters")
186
- raise err(400, "bad_path",
187
- "a harness path may use letters, digits, dot, dash, underscore and / only")
188
- return path
189
-
190
-
191
- def _blank(path, author, author_kind):
192
- return {"path": path, "versions": [], "trimmed": 0,
193
- "created": _now(), "createdBy": author, "createdKind": author_kind}
194
-
195
-
196
- def _append(record, body, author, author_kind, restored_from=None):
197
- """Append ONE version to a file record, in place, and report what was trimmed.
198
-
199
- The version NUMBER is monotonic and survives trimming — it counts writes, not stored entries.
200
- A version list whose numbers restart at 1 after a trim would make two different bodies share a
201
- name, and `restoredFrom` would then point at whichever one happened to be in the window.
202
- """
203
- # ⚠ A NEW LIST, NEVER `versions.append(...)` ON THE STORED ONE. `update()` hands the callback
204
- # the live document and may run it more than once; appending in place would then stack two
205
- # copies of the same version into the history on a retry.
206
- prior = record.get("versions") if isinstance(record.get("versions"), list) else []
207
- last = max((int(v.get("version") or 0) for v in prior if isinstance(v, dict)), default=0)
208
- entry = {"version": last + 1, "body": body, "author": author, "authorKind": author_kind,
209
- "created": _now(), "bytes": len(body.encode("utf-8"))}
210
- if restored_from:
211
- entry["restoredFrom"] = int(restored_from)
212
- versions = [*prior, entry]
213
- dropped = max(0, len(versions) - MAX_VERSIONS)
214
- if dropped:
215
- versions = versions[dropped:]
216
- record["versions"] = versions
217
- record["trimmed"] = int(record.get("trimmed") or 0) + dropped
218
- return entry
219
-
220
-
221
- def _head(record):
222
- """The newest version of a file record, or `None` for a record with no versions at all."""
223
- versions = record.get("versions") if isinstance(record.get("versions"), list) else []
224
- return versions[-1] if versions else None
225
-
226
-
227
- def _row(record):
228
- """One file, as the LIST door reports it: everything except the bodies.
229
-
230
- ⚠ NO BODY, AND THAT IS THE POINT. A list door that carried every version of every file would
231
- ship the whole harness on every pane render; the Harness tab lists first and opens one file
232
- second, which is exactly the shape this answers.
233
- """
234
- head = _head(record) or {}
235
- versions = record.get("versions") if isinstance(record.get("versions"), list) else []
236
- return {"path": record.get("path") or "",
237
- "version": int(head.get("version") or 0),
238
- "bytes": int(head.get("bytes") or 0),
239
- "author": head.get("author") or record.get("createdBy") or "",
240
- "authorKind": head.get("authorKind") or record.get("createdKind") or "user",
241
- "updated": head.get("created") or record.get("created") or "",
242
- "created": record.get("created") or "",
243
- "versions": len(versions),
244
- "trimmed": int(record.get("trimmed") or 0)}
245
-
246
-
247
- def _version_rows(record):
248
- """The history of one file, newest first, WITHOUT the bodies.
249
-
250
- A body per version is what makes a diff possible, and it is also what makes this payload big:
251
- 100 versions of a 128 KB file is 12 MB. The client asks for the two bodies it is diffing
252
- (`?path=…&version=N`), which is two round trips for a diff and none for a history list.
253
- """
254
- versions = record.get("versions") if isinstance(record.get("versions"), list) else []
255
- out = []
256
- for entry in reversed(versions):
257
- if not isinstance(entry, dict):
258
- continue
259
- row = {"version": int(entry.get("version") or 0),
260
- "author": entry.get("author") or "",
261
- "authorKind": entry.get("authorKind") or "user",
262
- "created": entry.get("created") or "",
263
- "bytes": int(entry.get("bytes") or 0)}
264
- if entry.get("restoredFrom"):
265
- row["restoredFrom"] = int(entry["restoredFrom"])
266
- out.append(row)
267
- return out
268
-
269
-
270
- def _limits():
271
- """The caps, IN the payload, so a client can say "this file is full" before a write fails.
272
-
273
- ⚠ A limit the client cannot see is a limit the user meets as an error. `trimmed` reports the
274
- one cap that acts without refusing; these report the three that refuse.
275
- """
276
- return {"maxBodyBytes": MAX_BODY_BYTES, "maxFiles": MAX_FILES,
277
- "maxVersions": MAX_VERSIONS, "maxPath": MAX_PATH}
278
-
279
-
280
- # ── the write door the SERVER uses (no session) ────────────────────────────────────────────────
281
- def record_version(runtime, agent_id, path, body, author="", author_kind="agent",
282
- restored_from=None):
283
- """Write one version from INSIDE the server — the agent's own half of R8.
284
-
285
- ⭐ THIS IS THE FUNCTION, NOT THE ROUTE, THAT MAKES "editable by BOTH the user and the agent"
286
- true. An agent acting inside an automation run holds no session and no cookie; if its only way
287
- to write were the HTTP door it would have to borrow a person's identity, and the authorship
288
- column would then be a record of who was logged in rather than of what wrote the file.
289
-
290
- Returns the appended entry. Raises nothing the caller cannot handle: an unknown agent is a
291
- `ValueError`, because a server-side caller has no HTTP response to put a 404 into.
292
- """
293
- agent_id = str(agent_id or "")
294
- if not _known_agent(runtime, agent_id):
295
- raise ValueError(f"no agent {agent_id!r} in this workspace")
296
- path = normalize_path(path)
297
- if path is None:
298
- raise ValueError("that is not an acceptable harness path")
299
- body = str(body or "")
300
- if len(body.encode("utf-8")) > MAX_BODY_BYTES:
301
- raise ValueError("harness body is over the size limit")
302
- kind = author_kind if author_kind in AUTHOR_KINDS else "agent"
303
- author = str(author or agent_id)
304
- # ⛔ THE FILE-COUNT CEILING IS CHECKED HERE, NOT INSIDE `_set`. An exception raised inside a
305
- # store `update` callback propagates out of a half-run read-modify-write, and the one thing a
306
- # refusal must never do is leave the caller unsure whether the write happened.
307
- existing = _files(runtime, agent_id)
308
- if path not in existing and len(existing) >= MAX_FILES:
309
- raise ValueError(f"this agent already has {MAX_FILES} harness files")
310
- appended = {}
311
-
312
- def _set(cur):
313
- cur = dict(cur or {})
314
- files = dict(cur.get(agent_id) or {}) if isinstance(cur.get(agent_id), dict) else {}
315
- record = dict(files[path]) if isinstance(files.get(path), dict) else _blank(path, author, kind)
316
- appended.clear()
317
- appended.update(_append(record, body, author, kind, restored_from))
318
- files[path] = record
319
- cur[agent_id] = files
320
- return cur
321
-
322
- runtime.update(HARNESS_KEY, _set, flush="sync")
323
- return appended
324
-
325
-
326
- # ── the routes ────────────────────────────────────────────────────────────────────────────────
327
- @router.get("/agents/{agent_id}/harness")
328
- def get_harness(agent_id: str, path: str = "", version: int = 0,
329
- session: Session = Depends(require_session)):
330
- """The list, one file, or one older body — decided by the query string (C4).
331
-
332
- ⚠ ADMIN-GATED LIKE EVERY OTHER AGENT DOOR (`routes_slack`), and for a stronger reason than
333
- consistency: a harness file is what the agent is INSTRUCTED to do, so writing one is closer to
334
- editing a permission than to editing a document.
335
- """
336
- agent_id = str(agent_id or "")
337
- agent_wall(session, agent_id)
338
- files = _files(session.runtime, agent_id)
339
- if not path:
340
- rows = [_row(rec) for _p, rec in sorted(files.items()) if isinstance(rec, dict)]
341
- return {"agent": agent_id, "files": rows, "limits": _limits()}
342
-
343
- wanted = _clean_path(path)
344
- record = files.get(wanted)
345
- if not isinstance(record, dict):
346
- raise err(404, "no_file", f"this agent has no harness file at {wanted}")
347
- versions = record.get("versions") if isinstance(record.get("versions"), list) else []
348
- if version:
349
- for entry in versions:
350
- if isinstance(entry, dict) and int(entry.get("version") or 0) == int(version):
351
- return {"agent": agent_id, **_row(record), "body": entry.get("body") or "",
352
- "atVersion": int(version), "history": _version_rows(record),
353
- "limits": _limits()}
354
- # ⛔ A TRIMMED VERSION IS A NAMED REFUSAL, NOT A 404 SHAPED LIKE A TYPO. The client asked
355
- # for something that existed and no longer does, and telling it apart from a bad number is
356
- # the difference between "roll back to v3" failing loudly and failing as if v3 never was.
357
- trimmed = int(record.get("trimmed") or 0)
358
- if trimmed and int(version) <= trimmed:
359
- raise err(410, "version_trimmed",
360
- f"version {int(version)} is older than the {MAX_VERSIONS} versions kept "
361
- f"for this file, and its body is gone")
362
- raise err(404, "no_version", f"this file has no version {int(version)}")
363
- head = _head(record) or {}
364
- return {"agent": agent_id, **_row(record), "body": head.get("body") or "",
365
- "atVersion": int(head.get("version") or 0), "history": _version_rows(record),
366
- "limits": _limits()}
367
-
368
-
369
- @router.put("/agents/{agent_id}/harness")
370
- def put_harness(agent_id: str, body: dict = Body(default=None),
371
- session: Session = Depends(require_session)):
372
- """Write a NEW version of one harness file. `{path, body, authorKind?, restoredFrom?}` (C4).
373
-
374
- ⛔ THERE IS NO OVERWRITE HERE AND THERE IS NO EDIT-IN-PLACE. R8's "every version kept" is not a
375
- UI affordance; it is this function refusing to have a code path that replaces a body. A
376
- roll-back arrives as an ordinary write carrying `restoredFrom`, so the history records that
377
- somebody went back rather than pretending the intervening versions never happened.
378
- """
379
- agent_id = str(agent_id or "")
380
- agent_wall(session, agent_id)
381
- body = body if isinstance(body, dict) else {}
382
- path = _clean_path(body.get("path"))
383
- text = body.get("body")
384
- if not isinstance(text, str):
385
- raise err(400, "no_body", "a harness file needs a body, even an empty one")
386
- if len(text.encode("utf-8")) > MAX_BODY_BYTES:
387
- raise err(413, "body_too_long",
388
- f"a harness file is at most {MAX_BODY_BYTES // 1024} KB; this one is larger")
389
-
390
- # ⚠ THE CALLER DECLARES THE PROVENANCE AND THE SERVER STAMPS THE IDENTITY. `authorKind` is a
391
- # claim about who WROTE the text (the person, or the agent in the chat panel); `author` is the
392
- # session and is never read off the request. Nothing is bought by forging the first — both
393
- # principals may write (R8) — and everything would be bought by forging the second.
394
- kind = str(body.get("authorKind") or "user").strip().lower()
395
- if kind not in AUTHOR_KINDS:
396
- raise err(400, "bad_author_kind", "authorKind is either user or agent")
397
- restored = body.get("restoredFrom")
398
- try:
399
- restored = int(restored) if restored else None
400
- except (TypeError, ValueError):
401
- raise err(400, "bad_version", "restoredFrom must be a version number")
402
-
403
- files = _files(session.runtime, agent_id)
404
- if path not in files and len(files) >= MAX_FILES:
405
- raise err(409, "too_many_files",
406
- f"this agent already has {MAX_FILES} harness files; delete one to add another")
407
- try:
408
- record_version(session.runtime, agent_id, path, text,
409
- author=session.uname, author_kind=kind, restored_from=restored)
410
- except ValueError as exc:
411
- raise err(400, "refused", str(exc))
412
-
413
- fresh = _files(session.runtime, agent_id).get(path)
414
- if not isinstance(fresh, dict) or not _head(fresh):
415
- # The store took the write and did not record it. A 200 here would tell an administrator
416
- # their skill file was saved when it was not — the shape `routes_slack` refuses too.
417
- raise err(503, "store_unavailable", "the harness file was NOT saved")
418
- head = _head(fresh)
419
- return {"agent": agent_id, **_row(fresh), "body": head.get("body") or "",
420
- "atVersion": int(head.get("version") or 0), "history": _version_rows(fresh),
421
- "limits": _limits()}
422
-
423
-
424
- @router.delete("/agents/{agent_id}/harness")
425
- def delete_harness(agent_id: str, path: str = "", session: Session = Depends(require_session)):
426
- """Drop one harness file AND its history.
427
-
428
- ⚠ THIS IS NOT THE THING R8 FORBIDS. R8 forbids a ROLL-BACK implemented as a delete — losing
429
- versions as a side effect of an edit. Deleting a file is a person deciding the file should not
430
- exist, which is a different act with a different button, and a store with no way to remove a
431
- file is one where a typo'd path is permanent.
432
- """
433
- agent_id = str(agent_id or "")
434
- agent_wall(session, agent_id)
435
- wanted = _clean_path(path)
436
- if wanted not in _files(session.runtime, agent_id):
437
- raise err(404, "no_file", f"this agent has no harness file at {wanted}")
438
-
439
- def _set(cur):
440
- cur = dict(cur or {})
441
- files = dict(cur.get(agent_id) or {}) if isinstance(cur.get(agent_id), dict) else {}
442
- files.pop(wanted, None)
443
- # An agent with no harness files leaves NO key behind. An empty dict per agent id is how a
444
- # bucket accumulates a row for every agent anybody ever opened the tab on.
445
- if files:
446
- cur[agent_id] = files
447
- else:
448
- cur.pop(agent_id, None)
449
- return cur
450
-
451
- session.runtime.update(HARNESS_KEY, _set, flush="sync")
452
- return {"agent": agent_id, "deleted": wanted,
453
- "files": [_row(rec) for _p, rec in sorted(_files(session.runtime, agent_id).items())
454
- if isinstance(rec, dict)],
455
- "limits": _limits()}
 
1
+ """routes_agent_harness.py — CONTRACT C4: the agent's HARNESS, kept as versioned files.
2
+
3
+ Owner item 3, verbatim (2026-08-18): *"This new module under agent is supposed to host any file
4
+ pertaining to the agent skills, router etc. So we build a user can build a custom harness for each
5
+ agent through the chat interface."*
6
+
7
+ GET /api/v1/agents/{id}/harness the file LIST (no bodies)
8
+ GET /api/v1/agents/{id}/harness?path=… one file: current body + its versions
9
+ GET /api/v1/agents/{id}/harness?path=…&version=N one older body, verbatim
10
+ PUT /api/v1/agents/{id}/harness write a NEW version of one file
11
+ DELETE /api/v1/agents/{id}/harness?path=… drop a file and its history
12
+
13
+ ⭐⭐ **R8 IS THE WHOLE SHAPE: BOTH PRINCIPALS WRITE, AND NOTHING IS EVER OVERWRITTEN.** A write
14
+ appends a version; a roll-back is a write of an older body (`restoredFrom`), never a delete. So the
15
+ history is a record of what happened rather than of what somebody last wanted it to look like.
16
+
17
+ ⛔ **R8 IS DELIBERATELY NOT R9.** An agent-authored ACTION's configuration is agent-only (item 8,
18
+ `routes_automation`); a harness file is not. Do not copy this file's posture over there or that
19
+ one's over here — the two rulings differ on purpose and they sit one screen apart in the product.
20
+
21
+ ⚠ **`author` IS THE SESSION, `authorKind` IS THE PROVENANCE, AND THEY ARE DIFFERENT FACTS.** Every
22
+ write through this router is made BY a signed-in administrator, so `author` is stamped from the
23
+ session and can never be supplied by the caller. `authorKind` says whether the BODY was drafted by
24
+ a person or by the agent in the chat — a claim the client is entitled to make, because both are
25
+ permitted (R8) and so nothing is bought by forging it. `record_version()` below is the server-side
26
+ door the automation engine uses when the agent writes with no session at all; that one stamps the
27
+ agent's own id as the author, which is the only case where `author` is not a username.
28
+
29
+ ⛔ **THE VERSION LIST IS CAPPED AND THE CAP IS REPORTED, NEVER SILENT.** `MAX_VERSIONS` versions of
30
+ one file are kept; past that the OLDEST are dropped and the count of what was dropped rides in
31
+ `trimmed` on every payload that mentions the file, so a reader can see that the history is partial
32
+ rather than infer that the file was only ever saved twice. This is the tenant document, which is
33
+ already 28.6 MB on tenant #0 and is deep-copied on every read: an unbounded per-agent history is
34
+ a store-sized leak with a UI in front of it.
35
+ """
36
+ from datetime import datetime, timezone
37
+
38
+ from fastapi import APIRouter, Body, Depends
39
+
40
+ from deps import Session, err, require_session
41
+
42
+ router = APIRouter(prefix="/api/v1")
43
+
44
+ #: The tenant's harness files: `{agent_id: {path: file_record}}`. A per-tenant bucket, so it rides
45
+ #: `runtime.store_key`'s prefix and never lands in tenant #0's namespace — the same rule
46
+ #: `routes_slack.AGENTS_KEY` follows for the agent records these hang off.
47
+ HARNESS_KEY = "agent_harness"
48
+
49
+ #: One body. Generous for a skill or a router file and far under the point where a single write
50
+ #: would move the tenant document measurably. A larger body is a REFUSAL at the door, not a
51
+ #: truncation: a silently truncated skill file is a harness that does not do what its text says.
52
+ MAX_BODY_BYTES = 128 * 1024
53
+
54
+ #: Files per agent. A refusal, not a trim — creating the 65th file is a different act from saving
55
+ #: the 101st version of one, and only the second can be a routine consequence of ordinary editing.
56
+ MAX_FILES = 64
57
+
58
+ #: Versions kept per file. Past this the oldest go and `trimmed` counts them (see the header).
59
+ MAX_VERSIONS = 100
60
+
61
+ MAX_PATH = 200
62
+
63
+ #: What a path may contain. ⛔ THIS IS NOT A FILESYSTEM PATH AND NOTHING HERE EVER TOUCHES A DISK —
64
+ #: the "files" are keys in a store bucket. The character rule exists so the key is displayable, is
65
+ #: safe to put in a URL, and cannot carry a traversal sequence that would look meaningful to a
66
+ #: future reader who assumes it IS a filesystem path. Fail-closed on the character set, not on a
67
+ #: list of forbidden sequences: an allow-list cannot be walked around by a spelling.
68
+ _PATH_OK = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-/")
69
+
70
+ AUTHOR_KINDS = ("user", "agent")
71
+
72
+
73
+ def _now():
74
+ return datetime.now(timezone.utc).isoformat(timespec="seconds")
75
+
76
+
77
+ def _all(rt):
78
+ """`{agent_id: {path: record}}` for one tenant. `{}` on any failure — an unreadable bucket must
79
+ degrade to "this agent has no harness files", never to a 500 on the pane that lists them."""
80
+ try:
81
+ found = rt.get(HARNESS_KEY) or {}
82
+ except Exception: # noqa: BLE001
83
+ return {}
84
+ return found if isinstance(found, dict) else {}
85
+
86
+
87
+ def _files(rt, agent_id):
88
+ found = _all(rt).get(str(agent_id))
89
+ return found if isinstance(found, dict) else {}
90
+
91
+
92
+ #: What kind of principal an id names. `None` = this tenant has no agent with that id.
93
+ #:
94
+ #: ⛔⛔ THERE ARE TWO AGENT REGISTRIES IN THIS PRODUCT AND THE FIRST DRAFT OF THIS FILE KNEW ONLY
95
+ #: ONE (ASK D-5, 2026-08-18). `routes_slack._agents` is the per-Slack-channel permission wall under
96
+ #: Manage users. The **Agents module** the owner's item 3 is about is the AUTOMATION surface —
97
+ #: `Shell.tsx` mounts `<Lazily surface="Agents"><AutomationSurface /></Lazily>`, and
98
+ #: `AutomationDetail`'s `panelTabs` is literally the `Properties | Run history` strip R7 adds
99
+ #: "Harness" to. Keyed on the Slack bucket alone, every `GET/PUT /agents/{id}/harness` from that
100
+ #: tab would have answered **404 no_agent**: whole, gate-green and dead on arrival
101
+ #: [[reachable-is-not-the-same-as-built]]. Verified independently before acting, not taken on
102
+ #: report: `surface="Agents"` mounts `AutomationSurface`, and `ManageAgentPane` contains ZERO
103
+ #: occurrences of Canvas, Properties, Run history or panelTabs.
104
+ #:
105
+ #: ⭐ ONE STORE, BOTH PRINCIPALS — never a second bucket keyed by surface. The owner's "agent
106
+ #: skills, router" files in two places is the parallel code path item 13 exists to refuse.
107
+ AGENT_SLACK = "slack"
108
+ AGENT_AUTOMATION = "automation"
109
+
110
+
111
+ def agent_kind(rt, agent_id):
112
+ """Which registry holds this id — `AGENT_SLACK`, `AGENT_AUTOMATION`, or `None`.
113
+
114
+ ⚠ A SYNTHETIC ROW IS NOT AN AGENT FOR THIS PURPOSE. `field:` and `system:` ids are DERIVED at
115
+ read time from a column definition or a connector schedule; they have no stored home, so a
116
+ harness file hung off one is an orphan the moment the column changes. `patch_automation`
117
+ already refuses those ids for the neighbouring reason, and this refuses them by simply not
118
+ finding them — `all_definitions` holds stored automations only.
119
+ """
120
+ import routes_slack
121
+ aid = str(agent_id or "")
122
+ if isinstance(routes_slack._agents(rt).get(aid), dict):
123
+ return AGENT_SLACK
124
+ import automation_engine as engine
125
+ try:
126
+ known = engine.all_definitions(rt) or {}
127
+ except Exception: # noqa: BLE001
128
+ return None
129
+ return AGENT_AUTOMATION if aid in known else None
130
+
131
+
132
+ def _known_agent(rt, agent_id):
133
+ """Does this tenant have an agent with this id, in EITHER registry?"""
134
+ return agent_kind(rt, agent_id) is not None
135
+
136
+
137
+ def agent_wall(session, agent_id):
138
+ """404 for an unknown id, else apply the wall THAT PRINCIPAL'S OWN SURFACE applies.
139
+
140
+ ⛔⛔ ONE DOOR, TWO WALLS, AND THAT IS NOT A SECOND CODE PATH — it is the refusal to invent a
141
+ THIRD wall. A Slack channel agent is administered under Manage users and every `/agents/*` door
142
+ in `routes_slack` is `admin_gate`; an automation lives in the Agents module and every door in
143
+ `routes_automation` is `module_gate("automation")`. A harness file is configuration OF the
144
+ agent it hangs off, so it is reached by whoever may already configure that agent. Picking one
145
+ of the two walls for both would either lock the Agents module's own users out of a tab the
146
+ owner asked for, or hand the Slack permission wall to anyone with an automation grant.
147
+ """
148
+ kind = agent_kind(session.runtime, agent_id)
149
+ if kind is None:
150
+ raise err(404, "no_agent", "there is no agent with that id in this workspace")
151
+ if kind == AGENT_SLACK:
152
+ import core.perms as perms
153
+ if not perms.is_admin(session.user):
154
+ raise err(403, "forbidden", "administrators only")
155
+ else:
156
+ session.require("automation")
157
+ return kind
158
+
159
+
160
+ def normalize_path(raw):
161
+ """THE path rule, in ONE place. Returns the cleaned path, or `None` if it is not acceptable.
162
+
163
+ ⛔⛔ ONE RULE, TWO DOORS, AND THAT IS WHY THIS IS A FUNCTION RATHER THAN TWO IF-BLOCKS. There
164
+ are two ways into this store — the HTTP route (which must answer 400) and `record_version()`
165
+ (which must raise `ValueError`, having no response to put a status into). Written twice, the
166
+ two copies are [[one-question-two-normalizers]] waiting to happen: the first weakening of one
167
+ copy is invisible because the other still refuses, so nothing goes red and the wall is now
168
+ half there. Written once, a change to the rule is felt at both doors and by the gate.
169
+ """
170
+ path = str(raw or "").strip().strip("/")
171
+ if not path or len(path) > MAX_PATH:
172
+ return None
173
+ if set(path) - _PATH_OK or ".." in path or "//" in path:
174
+ return None
175
+ return path
176
+
177
+
178
+ def _clean_path(raw):
179
+ """`normalize_path` at the HTTP door, where a refusal is a 400 with a reason."""
180
+ path = normalize_path(raw)
181
+ if path is None:
182
+ if not str(raw or "").strip().strip("/"):
183
+ raise err(400, "no_path", "a harness file needs a path, for example skills/router.md")
184
+ if len(str(raw)) > MAX_PATH:
185
+ raise err(400, "path_too_long", f"a harness path is at most {MAX_PATH} characters")
186
+ raise err(400, "bad_path",
187
+ "a harness path may use letters, digits, dot, dash, underscore and / only")
188
+ return path
189
+
190
+
191
+ def _blank(path, author, author_kind):
192
+ return {"path": path, "versions": [], "trimmed": 0,
193
+ "created": _now(), "createdBy": author, "createdKind": author_kind}
194
+
195
+
196
+ def _append(record, body, author, author_kind, restored_from=None):
197
+ """Append ONE version to a file record, in place, and report what was trimmed.
198
+
199
+ The version NUMBER is monotonic and survives trimming — it counts writes, not stored entries.
200
+ A version list whose numbers restart at 1 after a trim would make two different bodies share a
201
+ name, and `restoredFrom` would then point at whichever one happened to be in the window.
202
+ """
203
+ # ⚠ A NEW LIST, NEVER `versions.append(...)` ON THE STORED ONE. `update()` hands the callback
204
+ # the live document and may run it more than once; appending in place would then stack two
205
+ # copies of the same version into the history on a retry.
206
+ prior = record.get("versions") if isinstance(record.get("versions"), list) else []
207
+ last = max((int(v.get("version") or 0) for v in prior if isinstance(v, dict)), default=0)
208
+ entry = {"version": last + 1, "body": body, "author": author, "authorKind": author_kind,
209
+ "created": _now(), "bytes": len(body.encode("utf-8"))}
210
+ if restored_from:
211
+ entry["restoredFrom"] = int(restored_from)
212
+ versions = [*prior, entry]
213
+ dropped = max(0, len(versions) - MAX_VERSIONS)
214
+ if dropped:
215
+ versions = versions[dropped:]
216
+ record["versions"] = versions
217
+ record["trimmed"] = int(record.get("trimmed") or 0) + dropped
218
+ return entry
219
+
220
+
221
+ def _head(record):
222
+ """The newest version of a file record, or `None` for a record with no versions at all."""
223
+ versions = record.get("versions") if isinstance(record.get("versions"), list) else []
224
+ return versions[-1] if versions else None
225
+
226
+
227
+ def _row(record):
228
+ """One file, as the LIST door reports it: everything except the bodies.
229
+
230
+ ⚠ NO BODY, AND THAT IS THE POINT. A list door that carried every version of every file would
231
+ ship the whole harness on every pane render; the Harness tab lists first and opens one file
232
+ second, which is exactly the shape this answers.
233
+ """
234
+ head = _head(record) or {}
235
+ versions = record.get("versions") if isinstance(record.get("versions"), list) else []
236
+ return {"path": record.get("path") or "",
237
+ "version": int(head.get("version") or 0),
238
+ "bytes": int(head.get("bytes") or 0),
239
+ "author": head.get("author") or record.get("createdBy") or "",
240
+ "authorKind": head.get("authorKind") or record.get("createdKind") or "user",
241
+ "updated": head.get("created") or record.get("created") or "",
242
+ "created": record.get("created") or "",
243
+ "versions": len(versions),
244
+ "trimmed": int(record.get("trimmed") or 0)}
245
+
246
+
247
+ def _version_rows(record):
248
+ """The history of one file, newest first, WITHOUT the bodies.
249
+
250
+ A body per version is what makes a diff possible, and it is also what makes this payload big:
251
+ 100 versions of a 128 KB file is 12 MB. The client asks for the two bodies it is diffing
252
+ (`?path=…&version=N`), which is two round trips for a diff and none for a history list.
253
+ """
254
+ versions = record.get("versions") if isinstance(record.get("versions"), list) else []
255
+ out = []
256
+ for entry in reversed(versions):
257
+ if not isinstance(entry, dict):
258
+ continue
259
+ row = {"version": int(entry.get("version") or 0),
260
+ "author": entry.get("author") or "",
261
+ "authorKind": entry.get("authorKind") or "user",
262
+ "created": entry.get("created") or "",
263
+ "bytes": int(entry.get("bytes") or 0)}
264
+ if entry.get("restoredFrom"):
265
+ row["restoredFrom"] = int(entry["restoredFrom"])
266
+ out.append(row)
267
+ return out
268
+
269
+
270
+ def _limits():
271
+ """The caps, IN the payload, so a client can say "this file is full" before a write fails.
272
+
273
+ ⚠ A limit the client cannot see is a limit the user meets as an error. `trimmed` reports the
274
+ one cap that acts without refusing; these report the three that refuse.
275
+ """
276
+ return {"maxBodyBytes": MAX_BODY_BYTES, "maxFiles": MAX_FILES,
277
+ "maxVersions": MAX_VERSIONS, "maxPath": MAX_PATH}
278
+
279
+
280
+ # ── the write door the SERVER uses (no session) ────────────────────────────────────────────────
281
+ def record_version(runtime, agent_id, path, body, author="", author_kind="agent",
282
+ restored_from=None):
283
+ """Write one version from INSIDE the server — the agent's own half of R8.
284
+
285
+ ⭐ THIS IS THE FUNCTION, NOT THE ROUTE, THAT MAKES "editable by BOTH the user and the agent"
286
+ true. An agent acting inside an automation run holds no session and no cookie; if its only way
287
+ to write were the HTTP door it would have to borrow a person's identity, and the authorship
288
+ column would then be a record of who was logged in rather than of what wrote the file.
289
+
290
+ Returns the appended entry. Raises nothing the caller cannot handle: an unknown agent is a
291
+ `ValueError`, because a server-side caller has no HTTP response to put a 404 into.
292
+ """
293
+ agent_id = str(agent_id or "")
294
+ if not _known_agent(runtime, agent_id):
295
+ raise ValueError(f"no agent {agent_id!r} in this workspace")
296
+ path = normalize_path(path)
297
+ if path is None:
298
+ raise ValueError("that is not an acceptable harness path")
299
+ body = str(body or "")
300
+ if len(body.encode("utf-8")) > MAX_BODY_BYTES:
301
+ raise ValueError("harness body is over the size limit")
302
+ kind = author_kind if author_kind in AUTHOR_KINDS else "agent"
303
+ author = str(author or agent_id)
304
+ # ⛔ THE FILE-COUNT CEILING IS CHECKED HERE, NOT INSIDE `_set`. An exception raised inside a
305
+ # store `update` callback propagates out of a half-run read-modify-write, and the one thing a
306
+ # refusal must never do is leave the caller unsure whether the write happened.
307
+ existing = _files(runtime, agent_id)
308
+ if path not in existing and len(existing) >= MAX_FILES:
309
+ raise ValueError(f"this agent already has {MAX_FILES} harness files")
310
+ appended = {}
311
+
312
+ def _set(cur):
313
+ cur = dict(cur or {})
314
+ files = dict(cur.get(agent_id) or {}) if isinstance(cur.get(agent_id), dict) else {}
315
+ record = dict(files[path]) if isinstance(files.get(path), dict) else _blank(path, author, kind)
316
+ appended.clear()
317
+ appended.update(_append(record, body, author, kind, restored_from))
318
+ files[path] = record
319
+ cur[agent_id] = files
320
+ return cur
321
+
322
+ runtime.update(HARNESS_KEY, _set, flush="sync")
323
+ return appended
324
+
325
+
326
+ # ── the routes ────────────────────────────────────────────────────────────────────────────────
327
+ @router.get("/agents/{agent_id}/harness")
328
+ def get_harness(agent_id: str, path: str = "", version: int = 0,
329
+ session: Session = Depends(require_session)):
330
+ """The list, one file, or one older body — decided by the query string (C4).
331
+
332
+ ⚠ ADMIN-GATED LIKE EVERY OTHER AGENT DOOR (`routes_slack`), and for a stronger reason than
333
+ consistency: a harness file is what the agent is INSTRUCTED to do, so writing one is closer to
334
+ editing a permission than to editing a document.
335
+ """
336
+ agent_id = str(agent_id or "")
337
+ agent_wall(session, agent_id)
338
+ files = _files(session.runtime, agent_id)
339
+ if not path:
340
+ rows = [_row(rec) for _p, rec in sorted(files.items()) if isinstance(rec, dict)]
341
+ return {"agent": agent_id, "files": rows, "limits": _limits()}
342
+
343
+ wanted = _clean_path(path)
344
+ record = files.get(wanted)
345
+ if not isinstance(record, dict):
346
+ raise err(404, "no_file", f"this agent has no harness file at {wanted}")
347
+ versions = record.get("versions") if isinstance(record.get("versions"), list) else []
348
+ if version:
349
+ for entry in versions:
350
+ if isinstance(entry, dict) and int(entry.get("version") or 0) == int(version):
351
+ return {"agent": agent_id, **_row(record), "body": entry.get("body") or "",
352
+ "atVersion": int(version), "history": _version_rows(record),
353
+ "limits": _limits()}
354
+ # ⛔ A TRIMMED VERSION IS A NAMED REFUSAL, NOT A 404 SHAPED LIKE A TYPO. The client asked
355
+ # for something that existed and no longer does, and telling it apart from a bad number is
356
+ # the difference between "roll back to v3" failing loudly and failing as if v3 never was.
357
+ trimmed = int(record.get("trimmed") or 0)
358
+ if trimmed and int(version) <= trimmed:
359
+ raise err(410, "version_trimmed",
360
+ f"version {int(version)} is older than the {MAX_VERSIONS} versions kept "
361
+ f"for this file, and its body is gone")
362
+ raise err(404, "no_version", f"this file has no version {int(version)}")
363
+ head = _head(record) or {}
364
+ return {"agent": agent_id, **_row(record), "body": head.get("body") or "",
365
+ "atVersion": int(head.get("version") or 0), "history": _version_rows(record),
366
+ "limits": _limits()}
367
+
368
+
369
+ @router.put("/agents/{agent_id}/harness")
370
+ def put_harness(agent_id: str, body: dict = Body(default=None),
371
+ session: Session = Depends(require_session)):
372
+ """Write a NEW version of one harness file. `{path, body, authorKind?, restoredFrom?}` (C4).
373
+
374
+ ⛔ THERE IS NO OVERWRITE HERE AND THERE IS NO EDIT-IN-PLACE. R8's "every version kept" is not a
375
+ UI affordance; it is this function refusing to have a code path that replaces a body. A
376
+ roll-back arrives as an ordinary write carrying `restoredFrom`, so the history records that
377
+ somebody went back rather than pretending the intervening versions never happened.
378
+ """
379
+ agent_id = str(agent_id or "")
380
+ agent_wall(session, agent_id)
381
+ body = body if isinstance(body, dict) else {}
382
+ path = _clean_path(body.get("path"))
383
+ text = body.get("body")
384
+ if not isinstance(text, str):
385
+ raise err(400, "no_body", "a harness file needs a body, even an empty one")
386
+ if len(text.encode("utf-8")) > MAX_BODY_BYTES:
387
+ raise err(413, "body_too_long",
388
+ f"a harness file is at most {MAX_BODY_BYTES // 1024} KB; this one is larger")
389
+
390
+ # ⚠ THE CALLER DECLARES THE PROVENANCE AND THE SERVER STAMPS THE IDENTITY. `authorKind` is a
391
+ # claim about who WROTE the text (the person, or the agent in the chat panel); `author` is the
392
+ # session and is never read off the request. Nothing is bought by forging the first — both
393
+ # principals may write (R8) — and everything would be bought by forging the second.
394
+ kind = str(body.get("authorKind") or "user").strip().lower()
395
+ if kind not in AUTHOR_KINDS:
396
+ raise err(400, "bad_author_kind", "authorKind is either user or agent")
397
+ restored = body.get("restoredFrom")
398
+ try:
399
+ restored = int(restored) if restored else None
400
+ except (TypeError, ValueError):
401
+ raise err(400, "bad_version", "restoredFrom must be a version number")
402
+
403
+ files = _files(session.runtime, agent_id)
404
+ if path not in files and len(files) >= MAX_FILES:
405
+ raise err(409, "too_many_files",
406
+ f"this agent already has {MAX_FILES} harness files; delete one to add another")
407
+ try:
408
+ record_version(session.runtime, agent_id, path, text,
409
+ author=session.uname, author_kind=kind, restored_from=restored)
410
+ except ValueError as exc:
411
+ raise err(400, "refused", str(exc))
412
+
413
+ fresh = _files(session.runtime, agent_id).get(path)
414
+ if not isinstance(fresh, dict) or not _head(fresh):
415
+ # The store took the write and did not record it. A 200 here would tell an administrator
416
+ # their skill file was saved when it was not — the shape `routes_slack` refuses too.
417
+ raise err(503, "store_unavailable", "the harness file was NOT saved")
418
+ head = _head(fresh)
419
+ return {"agent": agent_id, **_row(fresh), "body": head.get("body") or "",
420
+ "atVersion": int(head.get("version") or 0), "history": _version_rows(fresh),
421
+ "limits": _limits()}
422
+
423
+
424
+ @router.delete("/agents/{agent_id}/harness")
425
+ def delete_harness(agent_id: str, path: str = "", session: Session = Depends(require_session)):
426
+ """Drop one harness file AND its history.
427
+
428
+ ⚠ THIS IS NOT THE THING R8 FORBIDS. R8 forbids a ROLL-BACK implemented as a delete — losing
429
+ versions as a side effect of an edit. Deleting a file is a person deciding the file should not
430
+ exist, which is a different act with a different button, and a store with no way to remove a
431
+ file is one where a typo'd path is permanent.
432
+ """
433
+ agent_id = str(agent_id or "")
434
+ agent_wall(session, agent_id)
435
+ wanted = _clean_path(path)
436
+ if wanted not in _files(session.runtime, agent_id):
437
+ raise err(404, "no_file", f"this agent has no harness file at {wanted}")
438
+
439
+ def _set(cur):
440
+ cur = dict(cur or {})
441
+ files = dict(cur.get(agent_id) or {}) if isinstance(cur.get(agent_id), dict) else {}
442
+ files.pop(wanted, None)
443
+ # An agent with no harness files leaves NO key behind. An empty dict per agent id is how a
444
+ # bucket accumulates a row for every agent anybody ever opened the tab on.
445
+ if files:
446
+ cur[agent_id] = files
447
+ else:
448
+ cur.pop(agent_id, None)
449
+ return cur
450
+
451
+ session.runtime.update(HARNESS_KEY, _set, flush="sync")
452
+ return {"agent": agent_id, "deleted": wanted,
453
+ "files": [_row(rec) for _p, rec in sorted(_files(session.runtime, agent_id).items())
454
+ if isinstance(rec, dict)],
455
+ "limits": _limits()}
api/routes_alerts.py CHANGED
@@ -1,668 +1,668 @@
1
- """routes_alerts.py — the Alerts module (wave 20, owner item 25, contract C-ALERT).
2
-
3
- GET /api/v1/alerts -> {alerts:[...]}
4
- POST /api/v1/alerts <- {viewId, topic, label?}
5
- DELETE /api/v1/alerts/{alert_id}
6
- POST /api/v1/alerts/{alert_id}/run -> evaluate now (the pane's manual refresh)
7
- GET /api/v1/notifications -> {unread, items:[...]}
8
- POST /api/v1/notifications/read <- {ids:[...]|null, read?:bool}
9
-
10
- The semantics — an alert is a view plus a remembered matched set, a notification is a NEW
11
- ENTRANT, and the first evaluation seeds silently — live in `core.alerts` with the reasoning.
12
- This file owns the two things a route must: WHO may do it, and HOW the view gets evaluated.
13
-
14
- ⭐ **THE EVALUATION RUNS AS THE ALERT'S OWNER, NOT AS THE CALLER.** `_run_alert` builds the pool
15
- for `rec['owner']`, never for whoever tripped the write hook. Any other choice leaks: a
16
- full-access admin editing a cell would otherwise evaluate a BU-scoped user's alert over the whole
17
- book, and the notification would name customers that user may not see — a permission leak wearing
18
- a notification's clothes. The owner's own scope is the only correct basis for their alert.
19
-
20
- ⚠ **AN ALERT IS NOT A SECOND READ PATH.** It resolves rows through the same
21
- `routes_customers.grid_assembly` / `routes_tables.ut_assembly` the grid uses, so a row that an
22
- alert can see is by construction a row its owner could open. Re-implementing the filter here
23
- would be a second definition of "matches", and those two would drift.
24
- """
25
- import re
26
-
27
- from fastapi import APIRouter, Body, Depends
28
-
29
- import core.alerts as alerts
30
- from deps import Session, err, require_session
31
-
32
- router = APIRouter(prefix="/api/v1")
33
-
34
- #: The alert-bearing surfaces. `ut_` tables are admitted by prefix, like everywhere else.
35
- _TOPICS = ("customer", "product")
36
-
37
- # ── ⭐⭐ WAVE 32 · T20 · CONTRACT C3 — THE INBOX SHAPE, DERIVED ON READ ────────────────────────
38
- #
39
- # `GET /notifications` gains `subject`, `kind` and `target` per item (`read` was always there).
40
- #
41
- # ⛔ DERIVED, NEVER STORED, AND THAT IS THE WHOLE OF WHY THIS WAVE EXISTS. Stamping the three
42
- # keys onto the record at write time would give them to notifications minted AFTER the deploy and
43
- # to nothing else — every notification already sitting in every tenant's inbox would open nothing,
44
- # and the feature would be correct in the source and absent from the product
45
- # ([[a-migration-that-runs-on-the-next-write]], D-201). A read-side derivation reaches a
46
- # notification queued last month. It also keeps the store shape out of `core/alerts.py`, which is
47
- # another lane's file this wave — but that is the convenience, not the reason.
48
- #
49
- # ⚠ TWO PRODUCERS WRITE TWO SHAPES into one inbox, and the vocabulary below is what tells them
50
- # apart. `_queue` (a record ENTERED a watched view) sets `topic`+`viewId`. `notify()` sets
51
- # `topic='automation'` and puts the producer's key in `alertId`, leaving `viewId` empty. Deciding
52
- # here means the client branches on ONE field instead of re-deriving the same split.
53
- #
54
- # ⛔ **D-101 IS CLOSED HERE, BY SUBTRACTION.** There was a THIRD shape — `kind='automation_review'`
55
- # + `autoId`, a card arriving at a review stage — and its producer `notify_review` was deleted by
56
- # W27/R3 with the review lanes. `automation_engine.py`'s own tombstone (search `notify_review`)
57
- # records the 2026-08-12 sweep: **no `.py` file anywhere produces one**, while the client branch,
58
- # its route and three gate legs stayed fully alive. D-101's exit condition is *"the client review
59
- # branch is deleted in the same change as any remaining residue, OR `notify_review` gains its real
60
- # caller"* — the residue is zero, so the branch goes. It is not carried into the Inbox: a stored
61
- # review notification (if any survives in a tenant from the wave-23 era) derives as an ordinary
62
- # `alert` with no target, i.e. an honest unclickable row, which is correct — the board it pointed
63
- # at was deleted two waves ago.
64
-
65
- #: C3's `kind` vocabulary. Plain strings on the wire — the client must never union over them
66
- #: (alertsModel's wave-9 law: a client union turns "the server grew a kind" into a dropped row).
67
- NOTIF_KIND_ALERT = "alert"
68
- NOTIF_KIND_AUTOMATION = "automation"
69
- NOTIF_KIND_SHARE = "share"
70
-
71
- #: C3's `target.module` vocabulary, and the automation sub-selection.
72
- TARGET_MODULE_DATABASE = "database"
73
- TARGET_MODULE_AUTOMATION = "automation"
74
- TARGET_TAB_RUNS = "runs"
75
-
76
- #: The topic `notify()` carries for a SHARE (W32-T28 writes it; nothing does yet, and a kind with
77
- #: no producer is a string that reads as a feature — the reason this constant is named here and
78
- #: cited from `routes_shares` rather than typed twice).
79
- SHARE_TOPIC = "share"
80
-
81
- #: `core.alerts.notify`'s default topic for a run outcome. Mirrors `inboxModel.AUTOMATION_TOPIC`.
82
- AUTOMATION_TOPIC = "automation"
83
-
84
- _UT_TOPIC = re.compile(r"ut_[A-Za-z0-9_]+\Z")
85
-
86
-
87
- def route_for_topic(topic):
88
- """A grid SCOPE key -> the registry route that renders it, or None.
89
-
90
- ⛔ THE SAME TABLE AS `alertsModel.routeForTopic`, and the parity is GATED
91
- (`verify_alerts.py`'s vocabulary scan) rather than trusted. The two built-ins are the only
92
- pair that differ — the registry names the surface (`customer_data`) while the grid names the
93
- scope (`customer`) — so a topic passed through as a route sends every click to a page that
94
- does not exist. `None` for anything else: a target this product cannot resolve must be ABSENT
95
- rather than plausible, because an absent target renders as a row that does not pretend to be
96
- clickable, and a wrong one renders as a click that silently goes nowhere.
97
- """
98
- t = str(topic or "").strip()
99
- if t == "customer":
100
- return "customer_data"
101
- if t == "product":
102
- return "product_data"
103
- if _UT_TOPIC.match(t):
104
- return t
105
- return None
106
-
107
-
108
- def _refusal_code(exc):
109
- """The `error.code` an `HTTPException` raised by `deps.err()` carries, or `""`.
110
-
111
- ⭐ W32-T22. Four refusals travel up the assembly chain — `unknown_table` (404), `forbidden`
112
- (403), `window_required` (409) and `store_not_ready` (503) — and each already names its own
113
- cause. Anything that reduces all four to one word is throwing away the only information the
114
- reader could have acted on. Returns `""` for a plain exception, so a caller can tell
115
- "refused, and here is why" apart from "broke, and we do not know why".
116
- """
117
- detail = getattr(exc, "detail", None)
118
- if isinstance(detail, dict):
119
- inner = detail.get("error")
120
- if isinstance(inner, dict):
121
- return str(inner.get("code") or "")
122
- return ""
123
-
124
-
125
- def notification_view(item):
126
- """One STORED notification -> the shape the Inbox renders. PURE, and total.
127
-
128
- Never raises and never drops a row: an item it cannot classify comes back as an `alert` with
129
- no `target`, which the client renders as an unclickable row rather than hiding. An inbox that
130
- silently omits what it does not understand is the one failure a reader cannot detect.
131
- """
132
- if not isinstance(item, dict):
133
- return item
134
- topic = str(item.get("topic") or "").strip()
135
- alert_id = str(item.get("alertId") or "").strip()
136
-
137
- # ⛔ THE ID TEST IS HALF OF EVERY BRANCH, and it is the load-bearing half. A row whose topic
138
- # says `automation` but whose producer key never arrived (a truncated payload, a server
139
- # mid-deploy) would otherwise be handed a target naming NOTHING — a click that appears to work
140
- # and silently does not, which is this repo's most-repeated failure shape. Failing the test
141
- # drops it to the `alert` branch, where `route_for_topic` refuses out loud by answering None.
142
- if topic == AUTOMATION_TOPIC and alert_id:
143
- kind = NOTIF_KIND_AUTOMATION
144
- target = {"module": TARGET_MODULE_AUTOMATION, "id": alert_id, "tab": TARGET_TAB_RUNS}
145
- elif topic == SHARE_TOPIC and alert_id:
146
- # ⭐ W32-T28: the sharer writes `key=<the ROUTE to open>` and, for a shared VIEW,
147
- # `row_id=<the view to select>`.
148
- #
149
- # ⛔ `key` IS ALREADY A ROUTE, NOT A RAW OBJECT ID, and the first version of this got it
150
- # wrong in a way worth recording: a shared VIEW put the VIEW's id in `alertId`, so the
151
- # target read `{module: "database", id: "view_42"}` — an instruction to open a database
152
- # called `view_42`. It looked right in the payload and would have opened nothing. The
153
- # producer resolves the object to its topic and hands over the route; this branch only
154
- # shapes what it is given.
155
- kind = NOTIF_KIND_SHARE
156
- row_id = str(item.get("rowId") or "").strip()
157
- target = {"module": TARGET_MODULE_DATABASE, "id": alert_id,
158
- **({"tab": row_id} if row_id else {})}
159
- else:
160
- kind = NOTIF_KIND_ALERT
161
- route = route_for_topic(topic)
162
- view_id = str(item.get("viewId") or "").strip()
163
- target = None if route is None else (
164
- {"module": TARGET_MODULE_DATABASE, "id": route,
165
- **({"tab": view_id} if view_id else {})})
166
-
167
- # The email split: `subject` is the HEADER (what this is about — the alert, the automation,
168
- # the database), `label` stays the BODY (what happened — the record that entered, the run
169
- # summary). They were one field, which is why a notification read as a sentence with no
170
- # sender and the pane could not be laid out like mail.
171
- subject = str(item.get("alertLabel") or "").strip() or str(item.get("label") or "").strip()
172
- # ⚠ `kind` is OVERWRITTEN, not merged. There was one stored value (`automation_review`) and it
173
- # is D-101's dead one; leaving it through would give the client two vocabularies for one
174
- # question, which is the defect this wave's item 6 is about in a different file.
175
- # ⭐⭐ W33-T28 (owner: "the Inbox reads like email") — THE SENDER, WHICH DID NOT EXIST.
176
- #
177
- # ⛔ A `verifier` reading the finished wave-32 surface found that the row's sender POSITION was
178
- # occupied by `kindLabel(n.kind)` — the literals "Alert" / "Automation" / "Shared with you" —
179
- # i.e. a CATEGORY standing where a who belongs, and no sender field anywhere on the wire, in
180
- # the model or in the markup. Mail has a from. This is it.
181
- #
182
- # ⚠ IT IS DERIVED HERE, NOT STORED, FOR EVERY KIND BUT ONE — and the exception is the point.
183
- # An alert firing and an automation landing rows have no person behind them; their honest
184
- # sender is the machine that did it, named as the thing the reader recognises. A SHARE has a
185
- # real person, and only the producer knows who: `routes_shares.py` writes it as `actor` and
186
- # this reads it back. ⛔ It is NOT parsed out of the body prose ("<name> shared this with
187
- # you") — a sender recovered by regexing a sentence breaks the first time the sentence is
188
- # reworded, and it would break silently, in the header.
189
- #
190
- # ⚠ FALLS BACK, NEVER BLANK. A share queued BEFORE `actor` existed has none, and a row with an
191
- # empty from column reads as a broken inbox rather than as an old notification.
192
- actor = str(item.get("actor") or "").strip()
193
- if kind == NOTIF_KIND_SHARE:
194
- sender = actor or "A teammate"
195
- elif kind == NOTIF_KIND_AUTOMATION:
196
- # ⛔ "Agents", not "Automation" (W34-T40, corrected at QA 2026-08-17). The client's
197
- # `senderOf` already falls back to `AGENTS_MODULE_LABEL` — but `if (sent) return sent`
198
- # runs FIRST, so this server literal won and every actor-less automation notification
199
- # showed the retired module name in the inbox's From column.
200
- sender = actor or "Agents"
201
- else:
202
- sender = actor or "Alerts"
203
- out = {**item, "read": bool(item.get("read")), "kind": kind,
204
- "subject": subject or "Notification", "sender": sender}
205
- if target is not None:
206
- out["target"] = target
207
- return out
208
-
209
-
210
- def inbox_view(box):
211
- """`core.alerts.inbox()`'s answer, with every item put through {@link notification_view}.
212
-
213
- ⚠ `unread` IS NOT RECOUNTED. It is the ACCOUNT's number and `items` is one page of it; a
214
- recount here would make the badge a function of whatever this page happened to include, which
215
- is the exact defect `alertsModel.parseInbox`'s own header records from the other side.
216
- """
217
- if not isinstance(box, dict):
218
- return box
219
- items = box.get("items")
220
- if not isinstance(items, list):
221
- return box
222
- # ⭐⭐ W33-T28 / D-208 — THE SERVER'S CLOCK RIDES WITH THE PAGE, and it is what lets the client
223
- # render a mail-shaped stamp ("09:41" today, "Aug 12" beyond) instead of `2026-08-13 09:41`.
224
- #
225
- # ⛔ THE CLIENT MUST NOT READ ITS OWN CLOCK, which is D-208's exit condition word for word and
226
- # is why this key exists rather than a `new Date()` in the browser. `at` is sent as UTC WITH
227
- # its offset (D-18) precisely so every reader sees the same instant; deciding "is this today?"
228
- # against a browser clock would re-introduce the drift the offset exists to remove — a reader a
229
- # day ahead being told an event happened tomorrow [[date-window-vocabulary]]. Both operands
230
- # now come from the same machine.
231
- # ⚠ Same funnel as the enrichment, so the read door and the mark-read door cannot disagree —
232
- # the note two lines up records what happened last time only one of them was enriched.
233
- return {**box, "now": alerts._now_iso(),
234
- "items": [notification_view(n) for n in items]}
235
-
236
-
237
- def _view_by_id(g, view_id):
238
- """One saved view out of an assembly, by id. `None` when there is no such view.
239
-
240
- ⛔⛔ W33-T29 (owner: *"Alert me about new records"* answering "Something went wrong") — THIS
241
- FUNCTION EXISTS BECAUSE TWO CALL SITES BOTH WROTE `(g.get("views") or {}).get(view_id)`, AND
242
- `g["views"] IS A LIST`. `aios_grid.views_from_defs` returns `[{...}]`, `workspace_wire` passes
243
- it straight out and both `ut_assembly` and `grid_assembly` return it unchanged — so `.get` on
244
- it raises `AttributeError`, and `views_from_defs` always returns at least one element, so the
245
- `or {}` never fires. **It raised on EVERY call, on every topic, since wave 20.**
246
-
247
- ⛔ AND THE TWO SITES FAILED DIFFERENTLY, WHICH IS WHY ONLY ONE WAS EVER REPORTED. In
248
- `_require_filtered_view` the raise lands ABOVE the handler's own `try`, so it leaves as a bare
249
- FastAPI 500 and the client's `errorMessage` turns any 5xx into *"Something went wrong on our
250
- side"* — the exact sentence the owner reported (D-107's shape, again: an attribute error above
251
- the guard arrives as plain text rather than as our envelope). In `_evaluate` the identical
252
- line is swallowed by `/notifications`' `except Exception: continue`, so **every stored
253
- view-alert was silently dropped from the Inbox** and nobody had anything to report at all.
254
- One expression, one loud symptom and one silent one.
255
-
256
- ⚠ SO IT IS A FUNCTION, NOT TWO FIXED LINES. Two copies of "find the view" is what let one site
257
- be discussed for three waves while its twin went unnoticed [[one-question-two-normalizers]].
258
-
259
- ⚠ It accepts a dict too, and that is not defensive noise: `verify_alerts`' door fixture was
260
- keyed `{id: view}` — which is precisely why the gate was green while production raised on
261
- every call. The fixture is moving to the production shape in this same change, and tolerating
262
- both here means a caller that legitimately holds one cannot resurrect the bug.
263
- """
264
- want = str(view_id or "")
265
- if not want:
266
- return None
267
- views = (g or {}).get("views")
268
- if isinstance(views, dict):
269
- found = views.get(want)
270
- return found if isinstance(found, dict) else None
271
- if not isinstance(views, list):
272
- return None
273
- for v in views:
274
- if isinstance(v, dict) and str(v.get("id") or "") == want:
275
- return v
276
- return None
277
-
278
-
279
- def _topic_or_400(raw):
280
- topic = str(raw or "").strip().lower()
281
- if topic.startswith("ut_") or topic in _TOPICS:
282
- return topic
283
- raise err(400, "bad_topic", f"topic must be one of {', '.join(_TOPICS)} or a ut_ table")
284
-
285
-
286
- def _owner_session(session: Session, owner: str):
287
- """A `Session` for the alert's OWNER (see the module note on why the owner, not the caller).
288
-
289
- ⚠ `Session` exposes `uname`/`admin` as PROPERTIES derived from `user`, not as fields — so an
290
- owner session is built by swapping the `user` RECORD and letting both derive themselves. An
291
- earlier version passed `uname=`/`admin=` to the constructor, which would have raised on the
292
- first write hook of the wave; the properties are the single definition of who a session is,
293
- and going around them is how a session with an admin flag and a non-admin record exists.
294
-
295
- Returns None when the owner is gone or deactivated — their alerts then stop evaluating rather
296
- than evaluating as somebody else, which is the fail-closed direction.
297
- """
298
- import core.users as users
299
-
300
- if str(owner) == str(session.uname):
301
- return session
302
- rec = (users.registry() or {}).get(str(owner))
303
- if not isinstance(rec, dict) or not rec.get("active", True):
304
- return None
305
- # `_public` is THE definition of what a session may know about its own account (never a hash
306
- # or a salt) — the same one `routes_auth` uses. Building the dict by hand here would be a
307
- # second definition, and the one that leaks is always the copy.
308
- return Session(tenant=session.tenant, user=users._public(str(owner), rec),
309
- claims=session.claims, runtime=session.runtime)
310
-
311
-
312
- def _evaluate(session: Session, rec: dict, assemblies=None):
313
- """Resolve `rec`'s view over its topic AS THE ALERT'S OWNER, then fold the result in.
314
-
315
- ⭐⭐ W31-T24 — `assemblies` IS A PER-REQUEST MEMO, KEYED `(topic, owner)`, and it is the whole
316
- of this ticket's server half. `/notifications` re-evaluates EVERY alert inline on read and each
317
- one built a FULL assembly — the pool, the workspace, `rows_from_pool` over every row. Two
318
- alerts on one view built that table twice; ten built it ten times. Nothing dedupes them,
319
- because each `_evaluate` was a closed call.
320
- ⚠ `(topic, owner)` and not `topic`: the assembly is built as the alert's OWNER (see the module
321
- note — evaluating a BU-scoped user's alert on a full-access admin's pool is a permission leak
322
- wearing a notification's clothes), so two owners on one topic are two DIFFERENT tables and
323
- must never share an entry. Getting that key wrong is the one way this optimisation could leak.
324
- ⚠ Passing nothing keeps the old behaviour exactly, which is what the create/run doors want:
325
- they evaluate ONE alert and a memo for a single call is pure overhead.
326
- """
327
- import aios_grid
328
- from harness import filter_eval
329
-
330
- owner_sess = _owner_session(session, rec.get("owner"))
331
- if owner_sess is None:
332
- return {"skipped": "owner_unavailable"}
333
- topic = str(rec.get("topic") or "")
334
- memo_key = (topic, str(owner_sess.uname))
335
- g = assemblies.get(memo_key) if isinstance(assemblies, dict) else None
336
- if g is None:
337
- try:
338
- if topic.startswith("ut_"):
339
- from routes_tables import ut_assembly
340
- # ⛔ `consume_corrections=False`, and the default was a REAL BUG, not a tidy-up.
341
- # `ut_assembly` defaults it True, so every `/notifications` read CONSUMED the
342
- # one-shot field-name correction acks for every `ut_` topic that has an alert —
343
- # taking them from the `/workspace` refresh that exists to show them to the person
344
- # who made the edit. The customer branch below has always passed False; this one
345
- # inherited a default nobody re-read. An inbox poll must never consume a one-shot.
346
- g = ut_assembly(owner_sess, topic,
347
- storage_key=f"{owner_sess.tenant}:{topic}:{owner_sess.uname}",
348
- consume_corrections=False)
349
- else:
350
- from routes_customers import grid_assembly
351
- g = grid_assembly(owner_sess, scope=topic, consume_corrections=False)
352
- except Exception as e: # noqa: BLE001
353
- # ⭐ W32-T22 — SKIPPING IS FINE HERE; SKIPPING ANONYMOUSLY IS NOT. This one must not
354
- # raise (one bad alert cannot empty an inbox), so unlike `_require_filtered_view` it
355
- # keeps a blanket catch — but it now reports the refusal's OWN code where there is
356
- # one. `type(e).__name__` said `HTTPException` for four different causes, and
357
- # `lastError` is the only place a user ever learns why an alert stopped firing.
358
- #
359
- # ⚠ `with_rows=True` STAYS on this path, deliberately: unlike the create door, an
360
- # evaluation genuinely needs the rows to run the filter over. So an alert on a
361
- # read-through grid is created (T22) and then skips at evaluation with
362
- # `window_required` naming why — which is D-184's remaining half, and it is a
363
- # SENTENCE now rather than silence.
364
- return {"skipped": _refusal_code(e) or "unavailable", "detail": type(e).__name__}
365
- if isinstance(assemblies, dict):
366
- assemblies[memo_key] = g
367
-
368
- view = _view_by_id(g, rec.get("viewId"))
369
- if not isinstance(view, dict):
370
- # Deleted, or un-shared out from under the alert. Say so on the RECORD rather than
371
- # deleting the alert: an alert that silently vanishes is indistinguishable from one that
372
- # never fires, and the user cannot debug what is not there.
373
- return {"skipped": "view_missing"}
374
-
375
- # The SAME row build the grid and `/customers` use — `rows_from_pool` is what puts derived
376
- # and overlay values on a row. Evaluating a filter against raw pool dicts would silently
377
- # never match any condition on a user-created or measure column.
378
- #
379
- # ⛔⛔ AND "THE SAME ROW BUILD" WAS NOT TRUE, WHICH MADE EVERY ALERT ON A `ut_*` DATABASE BLIND
380
- # TO IMPORTED DATA. Found by a verifier driving one real assembly through both paths.
381
- #
382
- # `routes_tables.table_rows` — the grid the person is looking at — merges the DEFINITION rows
383
- # underneath the overlay ("base first, overlay wins"; that merge is itself the fix for owner
384
- # item 3, *"it all got reseted"*). `_evaluate` is a second copy of that read and never got it:
385
- # it handed `ws['overlays']` to `rows_from_pool` raw, so for a `ut_*` table every base cell
386
- # evaluated as BLANK. Measured on one assembly, same view, same rows:
387
- # rows_src state='unpaid' / 'paid'
388
- # _evaluate saw state='' / '' ⇐ every base cell blank
389
- # the GRID saw state='unpaid' / 'paid'
390
- # so `state eq unpaid` matched NOTHING while the view showed one row, and `state isEmpty`
391
- # matched EVERYTHING while the view showed none. **The alert did not merely miss rows — it
392
- # inverted.** End to end: a row whose value arrived by import, automation, paste or the create
393
- # door never fired; only a value typed as a hand EDIT did.
394
- #
395
- # ⚠ Scope, so nobody widens the fix past its cause: materialised `ut_*` tables are hit;
396
- # `customer`/`product` are not (their fields are `source: "odoo"` and read off `rows_src`);
397
- # `ut_odoo_*` never reaches here (`with_rows=True` refuses first and returns
398
- # `skipped: window_required`).
399
- # ⛔ ORDER IS LOAD-BEARING AND IS THE GRID'S: base underneath, overlay ON TOP. Inverting it
400
- # would let a stale definition value shadow an edit the user has just made — the same defect
401
- # `table_rows`' own note records, arriving from the other side.
402
- _ov = (g.get("ws") or {}).get("overlays") or {}
403
- _merged = {}
404
- for _r in g["rows_src"]:
405
- _pid = str(_r.get("pid"))
406
- _cells = {k: v for k, v in _r.items() if k != "pid"}
407
- _o = _ov.get(_pid)
408
- if isinstance(_o, dict):
409
- _cells.update(_o)
410
- _merged[_pid] = _cells
411
- rows = aios_grid.rows_from_pool(g["rows_src"], g["fields"], _merged,
412
- derived=g.get("derived"))
413
- config = view.get("config") or view
414
- ctx = filter_eval.EvalCtx(
415
- cohort_sets={str(k): {str(p) for p in (v.get("memberPids") or ())}
416
- for k, v in (g.get("lists") or {}).items() if isinstance(v, dict)},
417
- measure_sets=g.get("measure_sets") or {},
418
- today=g.get("today"))
419
- pids = filter_eval.visible_pids(config.get("filters") or [], rows, g["fields"], ctx,
420
- member_pids=config.get("memberPids"))
421
- labels = {str(r.get("pid")): str(r.get("name") or r.get("pid")) for r in rows}
422
- return alerts.evaluate(rec.get("id"), [str(p) for p in pids],
423
- labels=labels, partial=False, st=session.runtime)
424
-
425
-
426
- @router.get("/alerts")
427
- def list_alerts(session: Session = Depends(require_session)):
428
- return {"alerts": alerts.list_alerts(user=session.uname, is_admin=session.admin,
429
- st=session.runtime)}
430
-
431
-
432
- @router.post("/alerts")
433
- def create_alert(body: dict = Body(default=None), session: Session = Depends(require_session)):
434
- body = body or {}
435
- view_id = str(body.get("viewId") or "").strip()
436
- if not view_id:
437
- raise err(400, "bad_view", "an alert needs the id of the view it watches")
438
- topic = _topic_or_400(body.get("topic"))
439
- _require_filtered_view(session, topic, view_id)
440
- import uuid
441
- aid = f"al_{uuid.uuid4().hex[:12]}"
442
- rec = alerts.create(aid, view_id=view_id, topic=topic, owner=session.uname,
443
- label=body.get("label") or "", st=session.runtime)
444
- # SEED IMMEDIATELY, so the alert starts from "everything currently matching is old news".
445
- # Deferring this to the first write hook would mean the next edit announces the whole view.
446
- outcome = _evaluate(session, rec)
447
- return {"alert": {**rec, "seeded": True}, "first": outcome}
448
-
449
-
450
- def _require_filtered_view(session: Session, topic: str, view_id: str):
451
- """400 unless `view_id` exists on `topic` AND actually narrows something.
452
-
453
- ⛔ AN ALERT ON AN UNFILTERED VIEW IS SILENTLY INCAPABLE OF ALERTING, which is worse than one
454
- that is refused. `filter_eval` treats an inactive tree as "no narrowing, every row shows"
455
- (`visible_pids`'s own rule), so such an alert seeds with the entire table and can never see an
456
- entrant again — there is nothing left to enter. The owner's words are *"when a Record gets
457
- into that Filter's criteria"*: no criteria, no alert, and said at creation rather than
458
- discovered by never being notified.
459
-
460
- `is_rule_active` is the SAME activeness predicate the engine and the column tints use — a
461
- half-typed rule is not a filter, and this must agree with what actually narrows or it would
462
- accept a view whose one rule the engine then ignores.
463
-
464
- ⭐⭐ WAVE 32 · T22 (owner item 17) — THIS FUNCTION WAS THE ERROR. Two defects, stacked, and
465
- the second one hid the first.
466
-
467
- (1) **IT ASKED FOR EVERY ROW OF A TABLE IT NEVER LOOKS AT.** The only thing read below is
468
- `g["views"]`. `ut_assembly` defaults `with_rows=True`, so creating an alert on a
469
- read-through grid built the whole pool — and `scoped_pool` refuses that with
470
- `409 window_required` over 963,783 rows, exactly as it is supposed to. `with_rows=False`
471
- (W31-T20's flag, built for precisely this) answers the same question with `scoped_pids`,
472
- runs the SAME `_defn_or_refuse` wall, and does not refuse. **That is D-184's create half,
473
- closed** — an alert on a read-through grid can now be made at all.
474
- (2) **A BLANKET `except Exception` TURNED EVERY NAMED REFUSAL INTO A 503.** `HTTPException`
475
- is an `Exception`, so `404 unknown_table`, `403 forbidden`, `409 window_required` and
476
- `503 store_not_ready` — four refusals that each say what is wrong — were all replaced by
477
- *"the table is unavailable — try again in a moment"*. ⛔ AND THAT SENTENCE NEVER REACHED
478
- A USER EITHER: `alertsApi.errorMessage` discards the text of any status ≥ 500 by design
479
- (a 5xx body is the server's internals), substituting *"Something went wrong on our
480
- side."* — which is the owner's screenshot, word for word. A knowable cause returned as a
481
- 5xx is invisible by construction, so re-wording the 503 could never have fixed this.
482
- ⚠ The except is narrowed, not deleted: an UNEXPECTED failure is still a 503, because that is
483
- honest. What it may no longer do is catch a refusal that already knows its own name.
484
- """
485
- from fastapi import HTTPException
486
-
487
- from harness import filter_eval
488
-
489
- try:
490
- if topic.startswith("ut_"):
491
- from routes_tables import ut_assembly
492
- # ⚠ `consume_corrections=False` — the customer branch has always passed it and this
493
- # one inherited a default nobody re-read. Creating an alert must not eat the one-shot
494
- # field-name correction acks belonging to the `/workspace` refresh that exists to show
495
- # them to the person who made the edit. Same defect `_evaluate`'s header records.
496
- g = ut_assembly(session, topic,
497
- storage_key=f"{session.tenant}:{topic}:{session.uname}",
498
- consume_corrections=False, with_rows=False)
499
- else:
500
- from routes_customers import grid_assembly
501
- g = grid_assembly(session, scope=topic, consume_corrections=False)
502
- except HTTPException:
503
- raise # it already names its own cause
504
- except Exception as e: # noqa: BLE001
505
- # Genuinely unexpected. Still a 503, and now it carries the exception TYPE — without it,
506
- # the one path that reaches this branch is also the one path with nothing to debug from.
507
- raise err(503, "unavailable",
508
- f"the table could not be read ({type(e).__name__}) — try again in a moment")
509
- view = _view_by_id(g, view_id)
510
- if not isinstance(view, dict):
511
- raise err(404, "no_view", "that view does not exist on this table")
512
- nodes, _conj = filter_eval.tree_parts((view.get("config") or view).get("filters") or [])
513
-
514
- # ⛔⛔ WAVE 33 · T29 — **CORRECTION: THE BLOCK BELOW IS TRUE ABOUT THE CODE AND FALSE ABOUT
515
- # PRODUCTION, AND IT MUST BE READ SECOND.** It claims the missing-argument `TypeError` "IS
516
- # owner item 17" — the owner's *"Something went wrong"*. It was not, and it could not have
517
- # been: at `cbcf005`, the build the owner was using, the dict-read on `views` sat ~10 lines
518
- # ABOVE this call and raised `AttributeError` on EVERY request, so the walk never reached the
519
- # leaf and the arity bug was unreachable. `_view_by_id`'s own header records that fix.
520
- #
521
- # ⚠ WHY THE STALE PARAGRAPH STAYS RATHER THAN GETTING DELETED: the arity bug was real, the
522
- # fix was right, and the three reasons it hid are the most transferable thing in this file.
523
- # What was wrong is only its CLAIM TO BE THE CAUSE. Two comment blocks in one function each
524
- # naming themselves as the origin of the same screenshot are mutually exclusive, and the next
525
- # reader believes whichever they meet first — which is why this correction sits above rather
526
- # than below. Caught by a verifier that read the SHIPPED file at the deployed commit instead
527
- # of the working tree. [[grep-output-is-not-source]]
528
- #
529
- # ⛔⛔ WAVE 32 · T22 — **THE CALL BELOW WAS MISSING AN ARGUMENT** (and wave 32 believed, wrongly,
530
- # that this was owner item 17 — see the correction directly above).
531
- #
532
- # `is_rule_active(rule, columns)` takes TWO parameters (`harness/filter_sql.py`; every other
533
- # caller in the repo passes both). This one passed ONE, so the moment the walk reached a LEAF
534
- # rule it raised `TypeError: is_rule_active() missing 1 required positional argument`.
535
- #
536
- # ⚠ READ WHAT THAT MEANS BEFORE FIXING ANYTHING ELSE: the walk only reaches a leaf when the
537
- # view HAS a condition — and a view with a condition is the only kind an alert is allowed on.
538
- # A view with no filters yields an empty `nodes`, so `_any_active` returns False without ever
539
- # calling this, and the reader gets the honest 400 `no_filter`. **So the only path that
540
- # worked was the refusal path: "Alert me about new records" had never once created an alert
541
- # on a filtered view.** ⛔ And the raise lands OUTSIDE the `try` above, so it was not even the
542
- # 503 — it was a bare FastAPI 500, which `alertsApi.errorMessage` renders as *"Something went
543
- # wrong on our side. Try again in a moment."*, the owner's screenshot word for word.
544
- #
545
- # ⚠ THREE THINGS HID IT, and they are worth more than the fix. (1) Python does not check
546
- # arity until the line RUNS, and this line runs only on the success path of a feature whose
547
- # every test exercised its refusals. (2) The `no_filter` 400 above it is a real, correct,
548
- # well-tested refusal, so the door looked alive. (3) `verify_alerts.py` asserts the refusal
549
- # (`no_filter` reaches the user) and the transport — never a creation. A gate can be green,
550
- # thorough and honest about everything except the one path the feature exists for.
551
- #
552
- # `_columns_map` is the DEFINITION of fields -> the membership set `is_rule_active` looks a
553
- # column up in; building a second dict here would be a second answer to one question, which
554
- # is this wave's other headline defect in a different file. Its leading underscore is a real
555
- # smell and is BOOKED (PENDING, mailbox/C.md) rather than worked around.
556
- columns = filter_eval._columns_map(g.get("fields") or [])
557
-
558
- def _any_active(ns):
559
- for n in ns or ():
560
- if isinstance(n, dict) and isinstance(n.get("children"), list):
561
- if _any_active(n["children"]):
562
- return True
563
- elif filter_eval.is_rule_active(n, columns):
564
- return True
565
- return False
566
-
567
- if not _any_active(nodes):
568
- raise err(400, "no_filter",
569
- "this view has no active filter, so no record can ever ENTER it — add a "
570
- "condition to the view first, then create the alert")
571
-
572
-
573
- @router.delete("/alerts/{alert_id}")
574
- def delete_alert(alert_id: str, session: Session = Depends(require_session)):
575
- rec = next((r for r in alerts.list_alerts(st=session.runtime)
576
- if str(r.get("id")) == str(alert_id)), None)
577
- if rec is None:
578
- raise err(404, "no_alert", "that alert does not exist")
579
- if str(rec.get("owner")) != str(session.uname) and not session.admin:
580
- raise err(403, "not_yours", "only the alert's owner (or an administrator) can delete it")
581
- alerts.delete(alert_id, st=session.runtime)
582
- return {"ok": True}
583
-
584
-
585
- @router.post("/alerts/{alert_id}/run")
586
- def run_alert(alert_id: str, session: Session = Depends(require_session)):
587
- rec = next((r for r in alerts.list_alerts(user=session.uname, is_admin=session.admin,
588
- st=session.runtime)
589
- if str(r.get("id")) == str(alert_id)), None)
590
- if rec is None:
591
- raise err(404, "no_alert", "that alert does not exist")
592
- return _evaluate(session, rec)
593
-
594
-
595
- @router.get("/notifications")
596
- def notifications(session: Session = Depends(require_session)):
597
- """The inbox — RE-EVALUATED on read, which is a deliberate design choice.
598
-
599
- ⭐ A-S1-2 RESOLVED THE OTHER WAY, and the reason is structural rather than a shortcut. The
600
- plan was a push hook: the automation engine calls `after_write` when it lands rows. But
601
- `run_async` runs on a BACKGROUND THREAD with no `Session` in scope, and an alert must be
602
- evaluated as its OWNER (see `_evaluate`) — so a push hook would have to mint a session inside
603
- a worker thread from a tenant runtime, which is exactly the kind of ad-hoc identity
604
- construction that leaks scope.
605
-
606
- Pulling on read has none of that: the caller IS a session, the assemblies are already
607
- scope-cached, and the user cannot observe the difference — an inbox is only ever read by
608
- someone opening it. The cost is that a notification is minted when you LOOK rather than when
609
- the row landed, so the `at` stamp is detection time, not arrival time.
610
-
611
- `after_write` stays exported for the day the engine can hand over a real identity.
612
-
613
- ⭐⭐ W31-T24 — ONE ASSEMBLY PER (TOPIC, OWNER), NOT ONE PER ALERT.
614
- ⛔ MEASURED FIRST, AND THE MEASUREMENT CORRECTS AN EARLIER READING OF IT. This route is
615
- **20 ms in-process and 3,280 ms live** on tenant #0 — but tenant #0 has **ZERO alerts**
616
- (censused 2026-08-12), so the 20 ms is an EMPTY LOOP and says nothing at all about what the
617
- re-evaluation costs. The live 3,280 ms is the two store reads either side of that loop. So the
618
- body below is not slow today; it is UNEXERCISED, and every alert a tenant creates adds a whole
619
- grid assembly to an inbox poll. The memo turns O(alerts) into O(distinct topic × owner), which
620
- is the difference between "fine" and "three seconds per alert" the day somebody uses the
621
- feature. ⚠ Making the read cheap by evaluating LESS is the obvious wrong fix and is not what
622
- this does: every alert is still evaluated, against the same rows, in the same order.
623
- """
624
- assemblies = {}
625
- for rec in alerts.list_alerts(user=session.uname, is_admin=False, st=session.runtime):
626
- try:
627
- _evaluate(session, rec, assemblies=assemblies)
628
- except Exception: # noqa: BLE001
629
- continue # one bad alert must not empty the pane
630
- # ⭐ W32-T20 (C3): every item leaves through `inbox_view`, so a notification queued before
631
- # this wave carries a `target` too. See `notification_view`'s header for why it is derived.
632
- return inbox_view(alerts.inbox(session.uname, st=session.runtime))
633
-
634
-
635
- @router.post("/notifications/read")
636
- def read_notifications(body: dict = Body(default=None),
637
- session: Session = Depends(require_session)):
638
- body = body or {}
639
- ids = body.get("ids")
640
- if ids is not None and not isinstance(ids, list):
641
- raise err(400, "bad_ids", "ids must be a list, or null to mark every notification")
642
- # ⚠ THE SAME ENRICHMENT ON BOTH DOORS. `mark_read` returns a fresh inbox, and the Inbox
643
- # module re-renders from it — an un-enriched answer here would strip `target` off every row
644
- # the moment somebody marked one read, i.e. the feature would work until first use.
645
- return inbox_view(alerts.mark_read(session.uname, ids, read=bool(body.get("read", True)),
646
- st=session.runtime))
647
-
648
-
649
- def after_write(session: Session, topic_key: str):
650
- """THE WRITE HOOK — call after a write that could change what a view matches.
651
-
652
- Exported as a plain function (not a route) so `core.grid_events`' callers and S2's automation
653
- upserts reach it the same way. It never raises: an alert evaluation failing must not fail the
654
- edit that triggered it.
655
-
656
- ⭐ W31-T24 — it shares `/notifications`' memo shape for the same reason: a write that changes
657
- one view can trip several alerts on the SAME topic, and each would otherwise rebuild the table.
658
- ⚠ STILL ZERO PRODUCTION CALLERS (W31-T24 confirmed it; the route docstring above says why the
659
- push hook was resolved the other way). Booked rather than wired: minting a session inside the
660
- engine's worker thread is the ad-hoc identity construction this file exists to avoid.
661
- """
662
- try:
663
- assemblies = {}
664
- return alerts.after_write(topic_key, st=session.runtime,
665
- runner=lambda rec: _evaluate(session, rec,
666
- assemblies=assemblies))
667
- except Exception: # noqa: BLE001
668
- return {"evaluated": 0}
 
1
+ """routes_alerts.py — the Alerts module (wave 20, owner item 25, contract C-ALERT).
2
+
3
+ GET /api/v1/alerts -> {alerts:[...]}
4
+ POST /api/v1/alerts <- {viewId, topic, label?}
5
+ DELETE /api/v1/alerts/{alert_id}
6
+ POST /api/v1/alerts/{alert_id}/run -> evaluate now (the pane's manual refresh)
7
+ GET /api/v1/notifications -> {unread, items:[...]}
8
+ POST /api/v1/notifications/read <- {ids:[...]|null, read?:bool}
9
+
10
+ The semantics — an alert is a view plus a remembered matched set, a notification is a NEW
11
+ ENTRANT, and the first evaluation seeds silently — live in `core.alerts` with the reasoning.
12
+ This file owns the two things a route must: WHO may do it, and HOW the view gets evaluated.
13
+
14
+ ⭐ **THE EVALUATION RUNS AS THE ALERT'S OWNER, NOT AS THE CALLER.** `_run_alert` builds the pool
15
+ for `rec['owner']`, never for whoever tripped the write hook. Any other choice leaks: a
16
+ full-access admin editing a cell would otherwise evaluate a BU-scoped user's alert over the whole
17
+ book, and the notification would name customers that user may not see — a permission leak wearing
18
+ a notification's clothes. The owner's own scope is the only correct basis for their alert.
19
+
20
+ ⚠ **AN ALERT IS NOT A SECOND READ PATH.** It resolves rows through the same
21
+ `routes_customers.grid_assembly` / `routes_tables.ut_assembly` the grid uses, so a row that an
22
+ alert can see is by construction a row its owner could open. Re-implementing the filter here
23
+ would be a second definition of "matches", and those two would drift.
24
+ """
25
+ import re
26
+
27
+ from fastapi import APIRouter, Body, Depends
28
+
29
+ import core.alerts as alerts
30
+ from deps import Session, err, require_session
31
+
32
+ router = APIRouter(prefix="/api/v1")
33
+
34
+ #: The alert-bearing surfaces. `ut_` tables are admitted by prefix, like everywhere else.
35
+ _TOPICS = ("customer", "product")
36
+
37
+ # ── ⭐⭐ WAVE 32 · T20 · CONTRACT C3 — THE INBOX SHAPE, DERIVED ON READ ────────────────────────
38
+ #
39
+ # `GET /notifications` gains `subject`, `kind` and `target` per item (`read` was always there).
40
+ #
41
+ # ⛔ DERIVED, NEVER STORED, AND THAT IS THE WHOLE OF WHY THIS WAVE EXISTS. Stamping the three
42
+ # keys onto the record at write time would give them to notifications minted AFTER the deploy and
43
+ # to nothing else — every notification already sitting in every tenant's inbox would open nothing,
44
+ # and the feature would be correct in the source and absent from the product
45
+ # ([[a-migration-that-runs-on-the-next-write]], D-201). A read-side derivation reaches a
46
+ # notification queued last month. It also keeps the store shape out of `core/alerts.py`, which is
47
+ # another lane's file this wave — but that is the convenience, not the reason.
48
+ #
49
+ # ⚠ TWO PRODUCERS WRITE TWO SHAPES into one inbox, and the vocabulary below is what tells them
50
+ # apart. `_queue` (a record ENTERED a watched view) sets `topic`+`viewId`. `notify()` sets
51
+ # `topic='automation'` and puts the producer's key in `alertId`, leaving `viewId` empty. Deciding
52
+ # here means the client branches on ONE field instead of re-deriving the same split.
53
+ #
54
+ # ⛔ **D-101 IS CLOSED HERE, BY SUBTRACTION.** There was a THIRD shape — `kind='automation_review'`
55
+ # + `autoId`, a card arriving at a review stage — and its producer `notify_review` was deleted by
56
+ # W27/R3 with the review lanes. `automation_engine.py`'s own tombstone (search `notify_review`)
57
+ # records the 2026-08-12 sweep: **no `.py` file anywhere produces one**, while the client branch,
58
+ # its route and three gate legs stayed fully alive. D-101's exit condition is *"the client review
59
+ # branch is deleted in the same change as any remaining residue, OR `notify_review` gains its real
60
+ # caller"* — the residue is zero, so the branch goes. It is not carried into the Inbox: a stored
61
+ # review notification (if any survives in a tenant from the wave-23 era) derives as an ordinary
62
+ # `alert` with no target, i.e. an honest unclickable row, which is correct — the board it pointed
63
+ # at was deleted two waves ago.
64
+
65
+ #: C3's `kind` vocabulary. Plain strings on the wire — the client must never union over them
66
+ #: (alertsModel's wave-9 law: a client union turns "the server grew a kind" into a dropped row).
67
+ NOTIF_KIND_ALERT = "alert"
68
+ NOTIF_KIND_AUTOMATION = "automation"
69
+ NOTIF_KIND_SHARE = "share"
70
+
71
+ #: C3's `target.module` vocabulary, and the automation sub-selection.
72
+ TARGET_MODULE_DATABASE = "database"
73
+ TARGET_MODULE_AUTOMATION = "automation"
74
+ TARGET_TAB_RUNS = "runs"
75
+
76
+ #: The topic `notify()` carries for a SHARE (W32-T28 writes it; nothing does yet, and a kind with
77
+ #: no producer is a string that reads as a feature — the reason this constant is named here and
78
+ #: cited from `routes_shares` rather than typed twice).
79
+ SHARE_TOPIC = "share"
80
+
81
+ #: `core.alerts.notify`'s default topic for a run outcome. Mirrors `inboxModel.AUTOMATION_TOPIC`.
82
+ AUTOMATION_TOPIC = "automation"
83
+
84
+ _UT_TOPIC = re.compile(r"ut_[A-Za-z0-9_]+\Z")
85
+
86
+
87
+ def route_for_topic(topic):
88
+ """A grid SCOPE key -> the registry route that renders it, or None.
89
+
90
+ ⛔ THE SAME TABLE AS `alertsModel.routeForTopic`, and the parity is GATED
91
+ (`verify_alerts.py`'s vocabulary scan) rather than trusted. The two built-ins are the only
92
+ pair that differ — the registry names the surface (`customer_data`) while the grid names the
93
+ scope (`customer`) — so a topic passed through as a route sends every click to a page that
94
+ does not exist. `None` for anything else: a target this product cannot resolve must be ABSENT
95
+ rather than plausible, because an absent target renders as a row that does not pretend to be
96
+ clickable, and a wrong one renders as a click that silently goes nowhere.
97
+ """
98
+ t = str(topic or "").strip()
99
+ if t == "customer":
100
+ return "customer_data"
101
+ if t == "product":
102
+ return "product_data"
103
+ if _UT_TOPIC.match(t):
104
+ return t
105
+ return None
106
+
107
+
108
+ def _refusal_code(exc):
109
+ """The `error.code` an `HTTPException` raised by `deps.err()` carries, or `""`.
110
+
111
+ ⭐ W32-T22. Four refusals travel up the assembly chain — `unknown_table` (404), `forbidden`
112
+ (403), `window_required` (409) and `store_not_ready` (503) — and each already names its own
113
+ cause. Anything that reduces all four to one word is throwing away the only information the
114
+ reader could have acted on. Returns `""` for a plain exception, so a caller can tell
115
+ "refused, and here is why" apart from "broke, and we do not know why".
116
+ """
117
+ detail = getattr(exc, "detail", None)
118
+ if isinstance(detail, dict):
119
+ inner = detail.get("error")
120
+ if isinstance(inner, dict):
121
+ return str(inner.get("code") or "")
122
+ return ""
123
+
124
+
125
+ def notification_view(item):
126
+ """One STORED notification -> the shape the Inbox renders. PURE, and total.
127
+
128
+ Never raises and never drops a row: an item it cannot classify comes back as an `alert` with
129
+ no `target`, which the client renders as an unclickable row rather than hiding. An inbox that
130
+ silently omits what it does not understand is the one failure a reader cannot detect.
131
+ """
132
+ if not isinstance(item, dict):
133
+ return item
134
+ topic = str(item.get("topic") or "").strip()
135
+ alert_id = str(item.get("alertId") or "").strip()
136
+
137
+ # ⛔ THE ID TEST IS HALF OF EVERY BRANCH, and it is the load-bearing half. A row whose topic
138
+ # says `automation` but whose producer key never arrived (a truncated payload, a server
139
+ # mid-deploy) would otherwise be handed a target naming NOTHING — a click that appears to work
140
+ # and silently does not, which is this repo's most-repeated failure shape. Failing the test
141
+ # drops it to the `alert` branch, where `route_for_topic` refuses out loud by answering None.
142
+ if topic == AUTOMATION_TOPIC and alert_id:
143
+ kind = NOTIF_KIND_AUTOMATION
144
+ target = {"module": TARGET_MODULE_AUTOMATION, "id": alert_id, "tab": TARGET_TAB_RUNS}
145
+ elif topic == SHARE_TOPIC and alert_id:
146
+ # ⭐ W32-T28: the sharer writes `key=<the ROUTE to open>` and, for a shared VIEW,
147
+ # `row_id=<the view to select>`.
148
+ #
149
+ # ⛔ `key` IS ALREADY A ROUTE, NOT A RAW OBJECT ID, and the first version of this got it
150
+ # wrong in a way worth recording: a shared VIEW put the VIEW's id in `alertId`, so the
151
+ # target read `{module: "database", id: "view_42"}` — an instruction to open a database
152
+ # called `view_42`. It looked right in the payload and would have opened nothing. The
153
+ # producer resolves the object to its topic and hands over the route; this branch only
154
+ # shapes what it is given.
155
+ kind = NOTIF_KIND_SHARE
156
+ row_id = str(item.get("rowId") or "").strip()
157
+ target = {"module": TARGET_MODULE_DATABASE, "id": alert_id,
158
+ **({"tab": row_id} if row_id else {})}
159
+ else:
160
+ kind = NOTIF_KIND_ALERT
161
+ route = route_for_topic(topic)
162
+ view_id = str(item.get("viewId") or "").strip()
163
+ target = None if route is None else (
164
+ {"module": TARGET_MODULE_DATABASE, "id": route,
165
+ **({"tab": view_id} if view_id else {})})
166
+
167
+ # The email split: `subject` is the HEADER (what this is about — the alert, the automation,
168
+ # the database), `label` stays the BODY (what happened — the record that entered, the run
169
+ # summary). They were one field, which is why a notification read as a sentence with no
170
+ # sender and the pane could not be laid out like mail.
171
+ subject = str(item.get("alertLabel") or "").strip() or str(item.get("label") or "").strip()
172
+ # ⚠ `kind` is OVERWRITTEN, not merged. There was one stored value (`automation_review`) and it
173
+ # is D-101's dead one; leaving it through would give the client two vocabularies for one
174
+ # question, which is the defect this wave's item 6 is about in a different file.
175
+ # ⭐⭐ W33-T28 (owner: "the Inbox reads like email") — THE SENDER, WHICH DID NOT EXIST.
176
+ #
177
+ # ⛔ A `verifier` reading the finished wave-32 surface found that the row's sender POSITION was
178
+ # occupied by `kindLabel(n.kind)` — the literals "Alert" / "Automation" / "Shared with you" —
179
+ # i.e. a CATEGORY standing where a who belongs, and no sender field anywhere on the wire, in
180
+ # the model or in the markup. Mail has a from. This is it.
181
+ #
182
+ # ⚠ IT IS DERIVED HERE, NOT STORED, FOR EVERY KIND BUT ONE — and the exception is the point.
183
+ # An alert firing and an automation landing rows have no person behind them; their honest
184
+ # sender is the machine that did it, named as the thing the reader recognises. A SHARE has a
185
+ # real person, and only the producer knows who: `routes_shares.py` writes it as `actor` and
186
+ # this reads it back. ⛔ It is NOT parsed out of the body prose ("<name> shared this with
187
+ # you") — a sender recovered by regexing a sentence breaks the first time the sentence is
188
+ # reworded, and it would break silently, in the header.
189
+ #
190
+ # ⚠ FALLS BACK, NEVER BLANK. A share queued BEFORE `actor` existed has none, and a row with an
191
+ # empty from column reads as a broken inbox rather than as an old notification.
192
+ actor = str(item.get("actor") or "").strip()
193
+ if kind == NOTIF_KIND_SHARE:
194
+ sender = actor or "A teammate"
195
+ elif kind == NOTIF_KIND_AUTOMATION:
196
+ # ⛔ "Agents", not "Automation" (W34-T40, corrected at QA 2026-08-17). The client's
197
+ # `senderOf` already falls back to `AGENTS_MODULE_LABEL` — but `if (sent) return sent`
198
+ # runs FIRST, so this server literal won and every actor-less automation notification
199
+ # showed the retired module name in the inbox's From column.
200
+ sender = actor or "Agents"
201
+ else:
202
+ sender = actor or "Alerts"
203
+ out = {**item, "read": bool(item.get("read")), "kind": kind,
204
+ "subject": subject or "Notification", "sender": sender}
205
+ if target is not None:
206
+ out["target"] = target
207
+ return out
208
+
209
+
210
+ def inbox_view(box):
211
+ """`core.alerts.inbox()`'s answer, with every item put through {@link notification_view}.
212
+
213
+ ⚠ `unread` IS NOT RECOUNTED. It is the ACCOUNT's number and `items` is one page of it; a
214
+ recount here would make the badge a function of whatever this page happened to include, which
215
+ is the exact defect `alertsModel.parseInbox`'s own header records from the other side.
216
+ """
217
+ if not isinstance(box, dict):
218
+ return box
219
+ items = box.get("items")
220
+ if not isinstance(items, list):
221
+ return box
222
+ # ⭐⭐ W33-T28 / D-208 — THE SERVER'S CLOCK RIDES WITH THE PAGE, and it is what lets the client
223
+ # render a mail-shaped stamp ("09:41" today, "Aug 12" beyond) instead of `2026-08-13 09:41`.
224
+ #
225
+ # ⛔ THE CLIENT MUST NOT READ ITS OWN CLOCK, which is D-208's exit condition word for word and
226
+ # is why this key exists rather than a `new Date()` in the browser. `at` is sent as UTC WITH
227
+ # its offset (D-18) precisely so every reader sees the same instant; deciding "is this today?"
228
+ # against a browser clock would re-introduce the drift the offset exists to remove — a reader a
229
+ # day ahead being told an event happened tomorrow [[date-window-vocabulary]]. Both operands
230
+ # now come from the same machine.
231
+ # ⚠ Same funnel as the enrichment, so the read door and the mark-read door cannot disagree —
232
+ # the note two lines up records what happened last time only one of them was enriched.
233
+ return {**box, "now": alerts._now_iso(),
234
+ "items": [notification_view(n) for n in items]}
235
+
236
+
237
+ def _view_by_id(g, view_id):
238
+ """One saved view out of an assembly, by id. `None` when there is no such view.
239
+
240
+ ⛔⛔ W33-T29 (owner: *"Alert me about new records"* answering "Something went wrong") — THIS
241
+ FUNCTION EXISTS BECAUSE TWO CALL SITES BOTH WROTE `(g.get("views") or {}).get(view_id)`, AND
242
+ `g["views"] IS A LIST`. `aios_grid.views_from_defs` returns `[{...}]`, `workspace_wire` passes
243
+ it straight out and both `ut_assembly` and `grid_assembly` return it unchanged — so `.get` on
244
+ it raises `AttributeError`, and `views_from_defs` always returns at least one element, so the
245
+ `or {}` never fires. **It raised on EVERY call, on every topic, since wave 20.**
246
+
247
+ ⛔ AND THE TWO SITES FAILED DIFFERENTLY, WHICH IS WHY ONLY ONE WAS EVER REPORTED. In
248
+ `_require_filtered_view` the raise lands ABOVE the handler's own `try`, so it leaves as a bare
249
+ FastAPI 500 and the client's `errorMessage` turns any 5xx into *"Something went wrong on our
250
+ side"* — the exact sentence the owner reported (D-107's shape, again: an attribute error above
251
+ the guard arrives as plain text rather than as our envelope). In `_evaluate` the identical
252
+ line is swallowed by `/notifications`' `except Exception: continue`, so **every stored
253
+ view-alert was silently dropped from the Inbox** and nobody had anything to report at all.
254
+ One expression, one loud symptom and one silent one.
255
+
256
+ ⚠ SO IT IS A FUNCTION, NOT TWO FIXED LINES. Two copies of "find the view" is what let one site
257
+ be discussed for three waves while its twin went unnoticed [[one-question-two-normalizers]].
258
+
259
+ ⚠ It accepts a dict too, and that is not defensive noise: `verify_alerts`' door fixture was
260
+ keyed `{id: view}` — which is precisely why the gate was green while production raised on
261
+ every call. The fixture is moving to the production shape in this same change, and tolerating
262
+ both here means a caller that legitimately holds one cannot resurrect the bug.
263
+ """
264
+ want = str(view_id or "")
265
+ if not want:
266
+ return None
267
+ views = (g or {}).get("views")
268
+ if isinstance(views, dict):
269
+ found = views.get(want)
270
+ return found if isinstance(found, dict) else None
271
+ if not isinstance(views, list):
272
+ return None
273
+ for v in views:
274
+ if isinstance(v, dict) and str(v.get("id") or "") == want:
275
+ return v
276
+ return None
277
+
278
+
279
+ def _topic_or_400(raw):
280
+ topic = str(raw or "").strip().lower()
281
+ if topic.startswith("ut_") or topic in _TOPICS:
282
+ return topic
283
+ raise err(400, "bad_topic", f"topic must be one of {', '.join(_TOPICS)} or a ut_ table")
284
+
285
+
286
+ def _owner_session(session: Session, owner: str):
287
+ """A `Session` for the alert's OWNER (see the module note on why the owner, not the caller).
288
+
289
+ ⚠ `Session` exposes `uname`/`admin` as PROPERTIES derived from `user`, not as fields — so an
290
+ owner session is built by swapping the `user` RECORD and letting both derive themselves. An
291
+ earlier version passed `uname=`/`admin=` to the constructor, which would have raised on the
292
+ first write hook of the wave; the properties are the single definition of who a session is,
293
+ and going around them is how a session with an admin flag and a non-admin record exists.
294
+
295
+ Returns None when the owner is gone or deactivated — their alerts then stop evaluating rather
296
+ than evaluating as somebody else, which is the fail-closed direction.
297
+ """
298
+ import core.users as users
299
+
300
+ if str(owner) == str(session.uname):
301
+ return session
302
+ rec = (users.registry() or {}).get(str(owner))
303
+ if not isinstance(rec, dict) or not rec.get("active", True):
304
+ return None
305
+ # `_public` is THE definition of what a session may know about its own account (never a hash
306
+ # or a salt) — the same one `routes_auth` uses. Building the dict by hand here would be a
307
+ # second definition, and the one that leaks is always the copy.
308
+ return Session(tenant=session.tenant, user=users._public(str(owner), rec),
309
+ claims=session.claims, runtime=session.runtime)
310
+
311
+
312
+ def _evaluate(session: Session, rec: dict, assemblies=None):
313
+ """Resolve `rec`'s view over its topic AS THE ALERT'S OWNER, then fold the result in.
314
+
315
+ ⭐⭐ W31-T24 — `assemblies` IS A PER-REQUEST MEMO, KEYED `(topic, owner)`, and it is the whole
316
+ of this ticket's server half. `/notifications` re-evaluates EVERY alert inline on read and each
317
+ one built a FULL assembly — the pool, the workspace, `rows_from_pool` over every row. Two
318
+ alerts on one view built that table twice; ten built it ten times. Nothing dedupes them,
319
+ because each `_evaluate` was a closed call.
320
+ ⚠ `(topic, owner)` and not `topic`: the assembly is built as the alert's OWNER (see the module
321
+ note — evaluating a BU-scoped user's alert on a full-access admin's pool is a permission leak
322
+ wearing a notification's clothes), so two owners on one topic are two DIFFERENT tables and
323
+ must never share an entry. Getting that key wrong is the one way this optimisation could leak.
324
+ ⚠ Passing nothing keeps the old behaviour exactly, which is what the create/run doors want:
325
+ they evaluate ONE alert and a memo for a single call is pure overhead.
326
+ """
327
+ import aios_grid
328
+ from harness import filter_eval
329
+
330
+ owner_sess = _owner_session(session, rec.get("owner"))
331
+ if owner_sess is None:
332
+ return {"skipped": "owner_unavailable"}
333
+ topic = str(rec.get("topic") or "")
334
+ memo_key = (topic, str(owner_sess.uname))
335
+ g = assemblies.get(memo_key) if isinstance(assemblies, dict) else None
336
+ if g is None:
337
+ try:
338
+ if topic.startswith("ut_"):
339
+ from routes_tables import ut_assembly
340
+ # ⛔ `consume_corrections=False`, and the default was a REAL BUG, not a tidy-up.
341
+ # `ut_assembly` defaults it True, so every `/notifications` read CONSUMED the
342
+ # one-shot field-name correction acks for every `ut_` topic that has an alert —
343
+ # taking them from the `/workspace` refresh that exists to show them to the person
344
+ # who made the edit. The customer branch below has always passed False; this one
345
+ # inherited a default nobody re-read. An inbox poll must never consume a one-shot.
346
+ g = ut_assembly(owner_sess, topic,
347
+ storage_key=f"{owner_sess.tenant}:{topic}:{owner_sess.uname}",
348
+ consume_corrections=False)
349
+ else:
350
+ from routes_customers import grid_assembly
351
+ g = grid_assembly(owner_sess, scope=topic, consume_corrections=False)
352
+ except Exception as e: # noqa: BLE001
353
+ # ⭐ W32-T22 — SKIPPING IS FINE HERE; SKIPPING ANONYMOUSLY IS NOT. This one must not
354
+ # raise (one bad alert cannot empty an inbox), so unlike `_require_filtered_view` it
355
+ # keeps a blanket catch — but it now reports the refusal's OWN code where there is
356
+ # one. `type(e).__name__` said `HTTPException` for four different causes, and
357
+ # `lastError` is the only place a user ever learns why an alert stopped firing.
358
+ #
359
+ # ⚠ `with_rows=True` STAYS on this path, deliberately: unlike the create door, an
360
+ # evaluation genuinely needs the rows to run the filter over. So an alert on a
361
+ # read-through grid is created (T22) and then skips at evaluation with
362
+ # `window_required` naming why — which is D-184's remaining half, and it is a
363
+ # SENTENCE now rather than silence.
364
+ return {"skipped": _refusal_code(e) or "unavailable", "detail": type(e).__name__}
365
+ if isinstance(assemblies, dict):
366
+ assemblies[memo_key] = g
367
+
368
+ view = _view_by_id(g, rec.get("viewId"))
369
+ if not isinstance(view, dict):
370
+ # Deleted, or un-shared out from under the alert. Say so on the RECORD rather than
371
+ # deleting the alert: an alert that silently vanishes is indistinguishable from one that
372
+ # never fires, and the user cannot debug what is not there.
373
+ return {"skipped": "view_missing"}
374
+
375
+ # The SAME row build the grid and `/customers` use — `rows_from_pool` is what puts derived
376
+ # and overlay values on a row. Evaluating a filter against raw pool dicts would silently
377
+ # never match any condition on a user-created or measure column.
378
+ #
379
+ # ⛔⛔ AND "THE SAME ROW BUILD" WAS NOT TRUE, WHICH MADE EVERY ALERT ON A `ut_*` DATABASE BLIND
380
+ # TO IMPORTED DATA. Found by a verifier driving one real assembly through both paths.
381
+ #
382
+ # `routes_tables.table_rows` — the grid the person is looking at — merges the DEFINITION rows
383
+ # underneath the overlay ("base first, overlay wins"; that merge is itself the fix for owner
384
+ # item 3, *"it all got reseted"*). `_evaluate` is a second copy of that read and never got it:
385
+ # it handed `ws['overlays']` to `rows_from_pool` raw, so for a `ut_*` table every base cell
386
+ # evaluated as BLANK. Measured on one assembly, same view, same rows:
387
+ # rows_src state='unpaid' / 'paid'
388
+ # _evaluate saw state='' / '' ⇐ every base cell blank
389
+ # the GRID saw state='unpaid' / 'paid'
390
+ # so `state eq unpaid` matched NOTHING while the view showed one row, and `state isEmpty`
391
+ # matched EVERYTHING while the view showed none. **The alert did not merely miss rows — it
392
+ # inverted.** End to end: a row whose value arrived by import, automation, paste or the create
393
+ # door never fired; only a value typed as a hand EDIT did.
394
+ #
395
+ # ⚠ Scope, so nobody widens the fix past its cause: materialised `ut_*` tables are hit;
396
+ # `customer`/`product` are not (their fields are `source: "odoo"` and read off `rows_src`);
397
+ # `ut_odoo_*` never reaches here (`with_rows=True` refuses first and returns
398
+ # `skipped: window_required`).
399
+ # ⛔ ORDER IS LOAD-BEARING AND IS THE GRID'S: base underneath, overlay ON TOP. Inverting it
400
+ # would let a stale definition value shadow an edit the user has just made — the same defect
401
+ # `table_rows`' own note records, arriving from the other side.
402
+ _ov = (g.get("ws") or {}).get("overlays") or {}
403
+ _merged = {}
404
+ for _r in g["rows_src"]:
405
+ _pid = str(_r.get("pid"))
406
+ _cells = {k: v for k, v in _r.items() if k != "pid"}
407
+ _o = _ov.get(_pid)
408
+ if isinstance(_o, dict):
409
+ _cells.update(_o)
410
+ _merged[_pid] = _cells
411
+ rows = aios_grid.rows_from_pool(g["rows_src"], g["fields"], _merged,
412
+ derived=g.get("derived"))
413
+ config = view.get("config") or view
414
+ ctx = filter_eval.EvalCtx(
415
+ cohort_sets={str(k): {str(p) for p in (v.get("memberPids") or ())}
416
+ for k, v in (g.get("lists") or {}).items() if isinstance(v, dict)},
417
+ measure_sets=g.get("measure_sets") or {},
418
+ today=g.get("today"))
419
+ pids = filter_eval.visible_pids(config.get("filters") or [], rows, g["fields"], ctx,
420
+ member_pids=config.get("memberPids"))
421
+ labels = {str(r.get("pid")): str(r.get("name") or r.get("pid")) for r in rows}
422
+ return alerts.evaluate(rec.get("id"), [str(p) for p in pids],
423
+ labels=labels, partial=False, st=session.runtime)
424
+
425
+
426
+ @router.get("/alerts")
427
+ def list_alerts(session: Session = Depends(require_session)):
428
+ return {"alerts": alerts.list_alerts(user=session.uname, is_admin=session.admin,
429
+ st=session.runtime)}
430
+
431
+
432
+ @router.post("/alerts")
433
+ def create_alert(body: dict = Body(default=None), session: Session = Depends(require_session)):
434
+ body = body or {}
435
+ view_id = str(body.get("viewId") or "").strip()
436
+ if not view_id:
437
+ raise err(400, "bad_view", "an alert needs the id of the view it watches")
438
+ topic = _topic_or_400(body.get("topic"))
439
+ _require_filtered_view(session, topic, view_id)
440
+ import uuid
441
+ aid = f"al_{uuid.uuid4().hex[:12]}"
442
+ rec = alerts.create(aid, view_id=view_id, topic=topic, owner=session.uname,
443
+ label=body.get("label") or "", st=session.runtime)
444
+ # SEED IMMEDIATELY, so the alert starts from "everything currently matching is old news".
445
+ # Deferring this to the first write hook would mean the next edit announces the whole view.
446
+ outcome = _evaluate(session, rec)
447
+ return {"alert": {**rec, "seeded": True}, "first": outcome}
448
+
449
+
450
+ def _require_filtered_view(session: Session, topic: str, view_id: str):
451
+ """400 unless `view_id` exists on `topic` AND actually narrows something.
452
+
453
+ ⛔ AN ALERT ON AN UNFILTERED VIEW IS SILENTLY INCAPABLE OF ALERTING, which is worse than one
454
+ that is refused. `filter_eval` treats an inactive tree as "no narrowing, every row shows"
455
+ (`visible_pids`'s own rule), so such an alert seeds with the entire table and can never see an
456
+ entrant again — there is nothing left to enter. The owner's words are *"when a Record gets
457
+ into that Filter's criteria"*: no criteria, no alert, and said at creation rather than
458
+ discovered by never being notified.
459
+
460
+ `is_rule_active` is the SAME activeness predicate the engine and the column tints use — a
461
+ half-typed rule is not a filter, and this must agree with what actually narrows or it would
462
+ accept a view whose one rule the engine then ignores.
463
+
464
+ ⭐⭐ WAVE 32 · T22 (owner item 17) — THIS FUNCTION WAS THE ERROR. Two defects, stacked, and
465
+ the second one hid the first.
466
+
467
+ (1) **IT ASKED FOR EVERY ROW OF A TABLE IT NEVER LOOKS AT.** The only thing read below is
468
+ `g["views"]`. `ut_assembly` defaults `with_rows=True`, so creating an alert on a
469
+ read-through grid built the whole pool — and `scoped_pool` refuses that with
470
+ `409 window_required` over 963,783 rows, exactly as it is supposed to. `with_rows=False`
471
+ (W31-T20's flag, built for precisely this) answers the same question with `scoped_pids`,
472
+ runs the SAME `_defn_or_refuse` wall, and does not refuse. **That is D-184's create half,
473
+ closed** — an alert on a read-through grid can now be made at all.
474
+ (2) **A BLANKET `except Exception` TURNED EVERY NAMED REFUSAL INTO A 503.** `HTTPException`
475
+ is an `Exception`, so `404 unknown_table`, `403 forbidden`, `409 window_required` and
476
+ `503 store_not_ready` — four refusals that each say what is wrong — were all replaced by
477
+ *"the table is unavailable — try again in a moment"*. ⛔ AND THAT SENTENCE NEVER REACHED
478
+ A USER EITHER: `alertsApi.errorMessage` discards the text of any status ≥ 500 by design
479
+ (a 5xx body is the server's internals), substituting *"Something went wrong on our
480
+ side."* — which is the owner's screenshot, word for word. A knowable cause returned as a
481
+ 5xx is invisible by construction, so re-wording the 503 could never have fixed this.
482
+ ⚠ The except is narrowed, not deleted: an UNEXPECTED failure is still a 503, because that is
483
+ honest. What it may no longer do is catch a refusal that already knows its own name.
484
+ """
485
+ from fastapi import HTTPException
486
+
487
+ from harness import filter_eval
488
+
489
+ try:
490
+ if topic.startswith("ut_"):
491
+ from routes_tables import ut_assembly
492
+ # ⚠ `consume_corrections=False` — the customer branch has always passed it and this
493
+ # one inherited a default nobody re-read. Creating an alert must not eat the one-shot
494
+ # field-name correction acks belonging to the `/workspace` refresh that exists to show
495
+ # them to the person who made the edit. Same defect `_evaluate`'s header records.
496
+ g = ut_assembly(session, topic,
497
+ storage_key=f"{session.tenant}:{topic}:{session.uname}",
498
+ consume_corrections=False, with_rows=False)
499
+ else:
500
+ from routes_customers import grid_assembly
501
+ g = grid_assembly(session, scope=topic, consume_corrections=False)
502
+ except HTTPException:
503
+ raise # it already names its own cause
504
+ except Exception as e: # noqa: BLE001
505
+ # Genuinely unexpected. Still a 503, and now it carries the exception TYPE — without it,
506
+ # the one path that reaches this branch is also the one path with nothing to debug from.
507
+ raise err(503, "unavailable",
508
+ f"the table could not be read ({type(e).__name__}) — try again in a moment")
509
+ view = _view_by_id(g, view_id)
510
+ if not isinstance(view, dict):
511
+ raise err(404, "no_view", "that view does not exist on this table")
512
+ nodes, _conj = filter_eval.tree_parts((view.get("config") or view).get("filters") or [])
513
+
514
+ # ⛔⛔ WAVE 33 · T29 — **CORRECTION: THE BLOCK BELOW IS TRUE ABOUT THE CODE AND FALSE ABOUT
515
+ # PRODUCTION, AND IT MUST BE READ SECOND.** It claims the missing-argument `TypeError` "IS
516
+ # owner item 17" — the owner's *"Something went wrong"*. It was not, and it could not have
517
+ # been: at `cbcf005`, the build the owner was using, the dict-read on `views` sat ~10 lines
518
+ # ABOVE this call and raised `AttributeError` on EVERY request, so the walk never reached the
519
+ # leaf and the arity bug was unreachable. `_view_by_id`'s own header records that fix.
520
+ #
521
+ # ⚠ WHY THE STALE PARAGRAPH STAYS RATHER THAN GETTING DELETED: the arity bug was real, the
522
+ # fix was right, and the three reasons it hid are the most transferable thing in this file.
523
+ # What was wrong is only its CLAIM TO BE THE CAUSE. Two comment blocks in one function each
524
+ # naming themselves as the origin of the same screenshot are mutually exclusive, and the next
525
+ # reader believes whichever they meet first — which is why this correction sits above rather
526
+ # than below. Caught by a verifier that read the SHIPPED file at the deployed commit instead
527
+ # of the working tree. [[grep-output-is-not-source]]
528
+ #
529
+ # ⛔⛔ WAVE 32 · T22 — **THE CALL BELOW WAS MISSING AN ARGUMENT** (and wave 32 believed, wrongly,
530
+ # that this was owner item 17 — see the correction directly above).
531
+ #
532
+ # `is_rule_active(rule, columns)` takes TWO parameters (`harness/filter_sql.py`; every other
533
+ # caller in the repo passes both). This one passed ONE, so the moment the walk reached a LEAF
534
+ # rule it raised `TypeError: is_rule_active() missing 1 required positional argument`.
535
+ #
536
+ # ⚠ READ WHAT THAT MEANS BEFORE FIXING ANYTHING ELSE: the walk only reaches a leaf when the
537
+ # view HAS a condition — and a view with a condition is the only kind an alert is allowed on.
538
+ # A view with no filters yields an empty `nodes`, so `_any_active` returns False without ever
539
+ # calling this, and the reader gets the honest 400 `no_filter`. **So the only path that
540
+ # worked was the refusal path: "Alert me about new records" had never once created an alert
541
+ # on a filtered view.** ⛔ And the raise lands OUTSIDE the `try` above, so it was not even the
542
+ # 503 — it was a bare FastAPI 500, which `alertsApi.errorMessage` renders as *"Something went
543
+ # wrong on our side. Try again in a moment."*, the owner's screenshot word for word.
544
+ #
545
+ # ⚠ THREE THINGS HID IT, and they are worth more than the fix. (1) Python does not check
546
+ # arity until the line RUNS, and this line runs only on the success path of a feature whose
547
+ # every test exercised its refusals. (2) The `no_filter` 400 above it is a real, correct,
548
+ # well-tested refusal, so the door looked alive. (3) `verify_alerts.py` asserts the refusal
549
+ # (`no_filter` reaches the user) and the transport — never a creation. A gate can be green,
550
+ # thorough and honest about everything except the one path the feature exists for.
551
+ #
552
+ # `_columns_map` is the DEFINITION of fields -> the membership set `is_rule_active` looks a
553
+ # column up in; building a second dict here would be a second answer to one question, which
554
+ # is this wave's other headline defect in a different file. Its leading underscore is a real
555
+ # smell and is BOOKED (PENDING, mailbox/C.md) rather than worked around.
556
+ columns = filter_eval._columns_map(g.get("fields") or [])
557
+
558
+ def _any_active(ns):
559
+ for n in ns or ():
560
+ if isinstance(n, dict) and isinstance(n.get("children"), list):
561
+ if _any_active(n["children"]):
562
+ return True
563
+ elif filter_eval.is_rule_active(n, columns):
564
+ return True
565
+ return False
566
+
567
+ if not _any_active(nodes):
568
+ raise err(400, "no_filter",
569
+ "this view has no active filter, so no record can ever ENTER it — add a "
570
+ "condition to the view first, then create the alert")
571
+
572
+
573
+ @router.delete("/alerts/{alert_id}")
574
+ def delete_alert(alert_id: str, session: Session = Depends(require_session)):
575
+ rec = next((r for r in alerts.list_alerts(st=session.runtime)
576
+ if str(r.get("id")) == str(alert_id)), None)
577
+ if rec is None:
578
+ raise err(404, "no_alert", "that alert does not exist")
579
+ if str(rec.get("owner")) != str(session.uname) and not session.admin:
580
+ raise err(403, "not_yours", "only the alert's owner (or an administrator) can delete it")
581
+ alerts.delete(alert_id, st=session.runtime)
582
+ return {"ok": True}
583
+
584
+
585
+ @router.post("/alerts/{alert_id}/run")
586
+ def run_alert(alert_id: str, session: Session = Depends(require_session)):
587
+ rec = next((r for r in alerts.list_alerts(user=session.uname, is_admin=session.admin,
588
+ st=session.runtime)
589
+ if str(r.get("id")) == str(alert_id)), None)
590
+ if rec is None:
591
+ raise err(404, "no_alert", "that alert does not exist")
592
+ return _evaluate(session, rec)
593
+
594
+
595
+ @router.get("/notifications")
596
+ def notifications(session: Session = Depends(require_session)):
597
+ """The inbox — RE-EVALUATED on read, which is a deliberate design choice.
598
+
599
+ ⭐ A-S1-2 RESOLVED THE OTHER WAY, and the reason is structural rather than a shortcut. The
600
+ plan was a push hook: the automation engine calls `after_write` when it lands rows. But
601
+ `run_async` runs on a BACKGROUND THREAD with no `Session` in scope, and an alert must be
602
+ evaluated as its OWNER (see `_evaluate`) — so a push hook would have to mint a session inside
603
+ a worker thread from a tenant runtime, which is exactly the kind of ad-hoc identity
604
+ construction that leaks scope.
605
+
606
+ Pulling on read has none of that: the caller IS a session, the assemblies are already
607
+ scope-cached, and the user cannot observe the difference — an inbox is only ever read by
608
+ someone opening it. The cost is that a notification is minted when you LOOK rather than when
609
+ the row landed, so the `at` stamp is detection time, not arrival time.
610
+
611
+ `after_write` stays exported for the day the engine can hand over a real identity.
612
+
613
+ ⭐⭐ W31-T24 — ONE ASSEMBLY PER (TOPIC, OWNER), NOT ONE PER ALERT.
614
+ ⛔ MEASURED FIRST, AND THE MEASUREMENT CORRECTS AN EARLIER READING OF IT. This route is
615
+ **20 ms in-process and 3,280 ms live** on tenant #0 — but tenant #0 has **ZERO alerts**
616
+ (censused 2026-08-12), so the 20 ms is an EMPTY LOOP and says nothing at all about what the
617
+ re-evaluation costs. The live 3,280 ms is the two store reads either side of that loop. So the
618
+ body below is not slow today; it is UNEXERCISED, and every alert a tenant creates adds a whole
619
+ grid assembly to an inbox poll. The memo turns O(alerts) into O(distinct topic × owner), which
620
+ is the difference between "fine" and "three seconds per alert" the day somebody uses the
621
+ feature. ⚠ Making the read cheap by evaluating LESS is the obvious wrong fix and is not what
622
+ this does: every alert is still evaluated, against the same rows, in the same order.
623
+ """
624
+ assemblies = {}
625
+ for rec in alerts.list_alerts(user=session.uname, is_admin=False, st=session.runtime):
626
+ try:
627
+ _evaluate(session, rec, assemblies=assemblies)
628
+ except Exception: # noqa: BLE001
629
+ continue # one bad alert must not empty the pane
630
+ # ⭐ W32-T20 (C3): every item leaves through `inbox_view`, so a notification queued before
631
+ # this wave carries a `target` too. See `notification_view`'s header for why it is derived.
632
+ return inbox_view(alerts.inbox(session.uname, st=session.runtime))
633
+
634
+
635
+ @router.post("/notifications/read")
636
+ def read_notifications(body: dict = Body(default=None),
637
+ session: Session = Depends(require_session)):
638
+ body = body or {}
639
+ ids = body.get("ids")
640
+ if ids is not None and not isinstance(ids, list):
641
+ raise err(400, "bad_ids", "ids must be a list, or null to mark every notification")
642
+ # ⚠ THE SAME ENRICHMENT ON BOTH DOORS. `mark_read` returns a fresh inbox, and the Inbox
643
+ # module re-renders from it — an un-enriched answer here would strip `target` off every row
644
+ # the moment somebody marked one read, i.e. the feature would work until first use.
645
+ return inbox_view(alerts.mark_read(session.uname, ids, read=bool(body.get("read", True)),
646
+ st=session.runtime))
647
+
648
+
649
+ def after_write(session: Session, topic_key: str):
650
+ """THE WRITE HOOK — call after a write that could change what a view matches.
651
+
652
+ Exported as a plain function (not a route) so `core.grid_events`' callers and S2's automation
653
+ upserts reach it the same way. It never raises: an alert evaluation failing must not fail the
654
+ edit that triggered it.
655
+
656
+ ⭐ W31-T24 — it shares `/notifications`' memo shape for the same reason: a write that changes
657
+ one view can trip several alerts on the SAME topic, and each would otherwise rebuild the table.
658
+ ⚠ STILL ZERO PRODUCTION CALLERS (W31-T24 confirmed it; the route docstring above says why the
659
+ push hook was resolved the other way). Booked rather than wired: minting a session inside the
660
+ engine's worker thread is the ad-hoc identity construction this file exists to avoid.
661
+ """
662
+ try:
663
+ assemblies = {}
664
+ return alerts.after_write(topic_key, st=session.runtime,
665
+ runner=lambda rec: _evaluate(session, rec,
666
+ assemblies=assemblies))
667
+ except Exception: # noqa: BLE001
668
+ return {"evaluated": 0}
api/routes_automation.py CHANGED
The diff for this file is too large to render. See raw diff
 
api/routes_customers.py CHANGED
The diff for this file is too large to render. See raw diff
 
api/routes_geo.py CHANGED
@@ -1,369 +1,369 @@
1
- """routes_geo.py — the MAP PROVIDER SEAM (wave 37, R3/R14, tickets T35/T36/T37).
2
-
3
- ⭐ WHY THIS FILE EXISTS AT ALL, and why the map does not simply call a vendor.
4
-
5
- R14 (owner, 2026-08-19): no Google Maps API key this wave. The map work is not dropped, it is
6
- retargeted to providers that need no key: OpenStreetMap tiles, OSRM routing, Nominatim geocoding.
7
- The deferral only holds if swapping a paid vendor back in is a CONFIG change rather than a rewrite,
8
- so every provider is a (base URL + attribution) pair read from the environment and served from ONE
9
- door. The renderer receives a URL template and a credit line; it knows nothing about who is behind
10
- them. That is the seam.
11
-
12
- tiles AIOS_MAP_TILE_URL default https://tile.openstreetmap.org/{z}/{x}/{y}.png
13
- geocode AIOS_GEOCODE_URL default https://nominatim.openstreetmap.org/search
14
- routing AIOS_ROUTE_URL default https://router.project-osrm.org/route/v1/driving/
15
-
16
- ⛔ R3'S ON-DEMAND RULE, AND WHY IT IS STRUCTURAL HERE RATHER THAN A COMMENT.
17
- R3 forbids geocoding a whole table automatically. Under R14 the reason moves from money to manners
18
- and gets sharper: Nominatim's usage policy allows roughly one request per second and explicitly
19
- forbids bulk harvesting, so a loop over 3,636 customers does not produce a bill, it produces a
20
- BLOCKED tenant. Three mechanisms enforce it, none of them advisory:
21
-
22
- 1. `/geo/geocode` takes an explicit LIST OF ADDRESSES from the caller. It cannot read a table, so
23
- there is no code path from "a database exists" to "its rows were geocoded". Somebody had to
24
- choose the records.
25
- 2. `MAX_BATCH` refuses an oversized list with a named error rather than truncating it. The client
26
- drives the loop and can therefore SHOW the wait, which is the half of the ticket a comment
27
- cannot satisfy.
28
- 3. `_throttle()` blocks in-process until the minimum interval has elapsed, so even a caller that
29
- ignores everything above cannot exceed the published rate.
30
-
31
- ⭐ AND A CACHE, because the policy asks for one: an address geocoded once is answered from memory
32
- for the rest of the container's life. It is the cheapest way to be a good citizen and it makes a
33
- re-run of the same records free rather than merely legal.
34
-
35
- ⚠ NOTHING HERE PROXIES TILES. Tiles are fetched by the BROWSER, straight from the provider, which
36
- is what every OSM client does and what the tile policy expects. Proxying them through this app
37
- would put a free tier in the path of every pan and would breach the same policy it looks like it is
38
- respecting. This door serves the tile URL, never the tile.
39
- """
40
- import os
41
- import re
42
- import threading
43
- import time
44
- from collections import OrderedDict
45
-
46
- import requests
47
- from fastapi import APIRouter, Body, Depends
48
-
49
- from deps import Session, err, require_session
50
-
51
- router = APIRouter(prefix="/api/v1")
52
-
53
- # --------------------------------------------------------------------- the seam
54
- #
55
- # Every value below is an environment override with a keyless default. Pointing the map at a paid
56
- # vendor is three variables on the Space and no code change, which is what makes R14 a deferral.
57
-
58
- _TILE_URL_DEFAULT = "https://tile.openstreetmap.org/{z}/{x}/{y}.png"
59
- _TILE_CREDIT_DEFAULT = "© OpenStreetMap contributors"
60
- _TILE_CREDIT_HREF_DEFAULT = "https://www.openstreetmap.org/copyright"
61
- _GEOCODE_URL_DEFAULT = "https://nominatim.openstreetmap.org/search"
62
- _ROUTE_URL_DEFAULT = "https://router.project-osrm.org/route/v1/driving/"
63
-
64
- #: Both policies require a real, identifying User-Agent. A generic one is how a shared service
65
- #: decides an application is a scraper, so this carries the product name and a contact URL.
66
- _UA_DEFAULT = "AIOS-Loopable/1.0 (+https://runloopable.com)"
67
-
68
- #: Nominatim publishes one request per second. 1100 ms leaves room for clock jitter rather than
69
- #: sitting exactly on the published edge.
70
- _MIN_INTERVAL_MS_DEFAULT = 1100
71
-
72
- #: The most addresses one call may carry. Small on purpose: the client loops and shows progress,
73
- #: and no single request can sit on the connection for a minute waiting out the throttle.
74
- MAX_BATCH = 25
75
-
76
- #: Tile zoom the provider actually serves. OSM stops at 19.
77
- _TILE_MAX_Z_DEFAULT = 19
78
-
79
- _CACHE_MAX = 4096
80
-
81
-
82
- def _env(name, default):
83
- return (os.environ.get(name) or "").strip() or default
84
-
85
-
86
- def _env_int(name, default):
87
- try:
88
- n = int((os.environ.get(name) or "").strip())
89
- return n if n > 0 else default
90
- except (TypeError, ValueError):
91
- return default
92
-
93
-
94
- def provider_config():
95
- """The whole seam as one dict. `GET /geo/providers` serves it and the gate reads it."""
96
- return {
97
- "tiles": {
98
- "url": _env("AIOS_MAP_TILE_URL", _TILE_URL_DEFAULT),
99
- "attribution": _env("AIOS_MAP_TILE_ATTRIBUTION", _TILE_CREDIT_DEFAULT),
100
- "attributionUrl": _env("AIOS_MAP_TILE_ATTRIBUTION_URL", _TILE_CREDIT_HREF_DEFAULT),
101
- "maxZoom": _env_int("AIOS_MAP_TILE_MAX_Z", _TILE_MAX_Z_DEFAULT),
102
- },
103
- "geocode": {
104
- "available": True,
105
- "maxBatch": MAX_BATCH,
106
- "minIntervalMs": _env_int("AIOS_GEOCODE_MIN_INTERVAL_MS", _MIN_INTERVAL_MS_DEFAULT),
107
- # ⭐⭐ THE NUMBER THE SURFACE MUST QUOTE, AND IT IS NOT THE RATE LIMIT.
108
- # T37 asks for the estimated wait to be shown BEFORE a run. The obvious source is
109
- # `minIntervalMs`, and it is wrong: 1.1 s is a floor on POLITENESS, not a prediction of
110
- # LATENCY. One real lookup was MEASURED at 3.28 s end to end on 2026-08-19. Quoting the
111
- # limit would promise a minute and take three, and a progress estimate that runs out
112
- # before the work does reads as a hang rather than as a wait.
113
- "secondsPerAddress": float(_env("AIOS_GEOCODE_SECONDS_EACH", "3.3")),
114
- "attribution": _env("AIOS_GEOCODE_ATTRIBUTION", _TILE_CREDIT_DEFAULT),
115
- },
116
- "route": {
117
- "available": True,
118
- "attribution": _env("AIOS_ROUTE_ATTRIBUTION", "Routing by OSRM"),
119
- },
120
- }
121
-
122
-
123
- # ------------------------------------------------------------------ the throttle
124
- #
125
- # ⚠ ONE lock and ONE timestamp for the whole process, shared by geocoding and routing, because the
126
- # rate limit belongs to the SERVICE and not to the endpoint. Two independent throttles would each
127
- # stay honest and together double the published rate.
128
-
129
- _gate = threading.Lock()
130
- _last_call = [0.0]
131
-
132
-
133
- def _throttle(min_interval_ms):
134
- """Block until the minimum interval since the last outbound call has elapsed.
135
-
136
- Returns the seconds actually waited, so a caller can report the wait rather than hide it."""
137
- wait = 0.0
138
- with _gate:
139
- gap = min_interval_ms / 1000.0
140
- now = time.monotonic()
141
- due = _last_call[0] + gap
142
- if now < due:
143
- wait = due - now
144
- time.sleep(wait)
145
- _last_call[0] = time.monotonic()
146
- return wait
147
-
148
-
149
- # --------------------------------------------------------------------- the cache
150
-
151
- _cache = OrderedDict()
152
- _cache_lock = threading.Lock()
153
-
154
-
155
- def _cache_key(q, country):
156
- return (re.sub(r"\s+", " ", str(q or "")).strip().lower(), str(country or "").strip().lower())
157
-
158
-
159
- def _cache_get(key):
160
- with _cache_lock:
161
- if key not in _cache:
162
- return None
163
- _cache.move_to_end(key)
164
- return _cache[key]
165
-
166
-
167
- def _cache_put(key, value):
168
- with _cache_lock:
169
- _cache[key] = value
170
- _cache.move_to_end(key)
171
- while len(_cache) > _CACHE_MAX:
172
- _cache.popitem(last=False)
173
-
174
-
175
- def geocode_one(address, country=None, timeout=12):
176
- """One address to {lat, lon, label} or None.
177
-
178
- ⛔ THE ONLY PLACE AN OUTBOUND GEOCODE HAPPENS. `ai_enrich.py`'s geocode field calls this rather
179
- than reaching for `requests` itself, so the throttle and the cache cannot be walked around by a
180
- second caller. A cache hit costs no request and no wait."""
181
- q = re.sub(r"\s+", " ", str(address or "")).strip()
182
- if not q:
183
- return None
184
- key = _cache_key(q, country)
185
- hit = _cache_get(key)
186
- if hit is not None:
187
- return dict(hit) if hit else None
188
-
189
- cfg = provider_config()["geocode"]
190
- _throttle(cfg["minIntervalMs"])
191
- params = {"q": q, "format": "jsonv2", "limit": 1}
192
- if country:
193
- params["countrycodes"] = str(country).strip().lower()
194
- try:
195
- r = requests.get(
196
- _env("AIOS_GEOCODE_URL", _GEOCODE_URL_DEFAULT),
197
- params=params,
198
- headers={"User-Agent": _env("AIOS_GEO_USER_AGENT", _UA_DEFAULT),
199
- "Accept": "application/json"},
200
- timeout=timeout,
201
- )
202
- r.raise_for_status()
203
- rows = r.json()
204
- except Exception:
205
- # ⚠ A transport failure is NOT cached. Caching it would turn one bad minute into a
206
- # permanently empty column that no re-run could ever repair.
207
- return None
208
- if not isinstance(rows, list) or not rows:
209
- # A genuine "no such place" IS cached, as a negative: asking again gets the same answer and
210
- # spends another second of a shared service's budget to hear it.
211
- _cache_put(key, {})
212
- return None
213
- top = rows[0] or {}
214
- try:
215
- lat = float(top.get("lat"))
216
- lon = float(top.get("lon"))
217
- except (TypeError, ValueError):
218
- _cache_put(key, {})
219
- return None
220
- if not (-90 <= lat <= 90) or not (-180 <= lon <= 180):
221
- _cache_put(key, {})
222
- return None
223
- out = {"lat": lat, "lon": lon, "label": str(top.get("display_name") or q)}
224
- _cache_put(key, out)
225
- return dict(out)
226
-
227
-
228
- # ---------------------------------------------------------------------- the doors
229
-
230
-
231
- @router.get("/geo/providers")
232
- def geo_providers(session: Session = Depends(require_session)):
233
- """What the map should draw with, and who to credit for it.
234
-
235
- Session gated like every other door here. The values are not secret, but an unauthenticated
236
- endpoint on this app is a door somebody eventually hangs something else on."""
237
- return provider_config()
238
-
239
-
240
- @router.post("/geo/geocode")
241
- def geo_geocode(body: dict = Body(...), session: Session = Depends(require_session)):
242
- """Turn a bounded list of addresses into coordinates.
243
-
244
- ⛔ Takes ADDRESSES, never a table key, a view id or a filter. There is deliberately no shape of
245
- request that means "geocode everything", which is R3 expressed as an API rather than as a
246
- warning."""
247
- items = body.get("addresses")
248
- # ⭐ THE SECOND SHAPE, AND IT EXISTS TO KEEP ONE COMPOSER. A caller may send whole RECORDS plus
249
- # the geocode field's config instead of finished address strings, and the server builds the
250
- # query with `ai_enrich.geocode_query`. The alternative was a TypeScript twin of that function
251
- # on the client, and an address composer that exists twice will disagree the first time either
252
- # copy learns about a new column ([[one-question-two-normalizers]]) -- which is exactly the trap
253
- # B's measured `New York (US)` suffix would spring, silently, in whichever copy forgot.
254
- if items is None:
255
- records = body.get("records")
256
- cfg = body.get("config") or {}
257
- if isinstance(records, list) and records:
258
- import ai_enrich
259
- items = []
260
- for rec in records:
261
- if not isinstance(rec, dict):
262
- continue
263
- q = ai_enrich.geocode_query(rec.get("row") or {}, cfg)
264
- # ⛔ A record with no usable address is REPORTED, not dropped: it comes back
265
- # `found: false` with an empty address, so the caller can name it on screen instead
266
- # of quietly returning fewer answers than it asked questions.
267
- items.append({"key": rec.get("key"), "address": q})
268
- if not body.get("country") and cfg.get("country"):
269
- body = {**body, "country": cfg.get("country")}
270
- if not isinstance(items, list) or not items:
271
- raise err(400, "no_addresses", "Send at least one address to look up.")
272
- if len(items) > MAX_BATCH:
273
- raise err(
274
- 400,
275
- "batch_too_large",
276
- f"Look up at most {MAX_BATCH} addresses per request. "
277
- f"The map sends them in batches of {MAX_BATCH} so the wait stays visible.",
278
- )
279
- country = body.get("country")
280
- out = []
281
- started = time.monotonic()
282
- for raw in items:
283
- if isinstance(raw, dict):
284
- key, addr = raw.get("key"), raw.get("address")
285
- else:
286
- key, addr = None, raw
287
- # ⛔ AN EMPTY QUERY COSTS NO REQUEST AND CARRIES ITS OWN REASON. Nominatim's policy is a
288
- # budget shared with everyone else using it, and asking it to place "" would spend a second
289
- # of that budget to be told nothing. `reason` is what lets the surface say "no address on
290
- # file" rather than "not found", which are different facts and want different actions.
291
- if not str(addr or "").strip():
292
- out.append({"key": key, "address": "", "found": False, "lat": None, "lon": None,
293
- "label": None, "reason": "no_address"})
294
- continue
295
- hit = geocode_one(addr, country=country)
296
- out.append({"key": key, "address": str(addr or ""), "found": bool(hit),
297
- "reason": None if hit else "not_found",
298
- "lat": hit["lat"] if hit else None,
299
- "lon": hit["lon"] if hit else None,
300
- "label": hit["label"] if hit else None})
301
- elapsed = time.monotonic() - started
302
- return {"results": out,
303
- "found": sum(1 for r in out if r["found"]),
304
- "asked": len(out),
305
- "secondsElapsed": round(elapsed, 2),
306
- "attribution": provider_config()["geocode"]["attribution"]}
307
-
308
-
309
- @router.post("/geo/route")
310
- def geo_route(body: dict = Body(...), session: Session = Depends(require_session)):
311
- """Road distance, duration and the drawn line, for stops ALREADY put in order.
312
-
313
- ⭐ THE ORDERING IS NOT DONE HERE. `mapProjection.planRoute` sequences the stops on the client,
314
- for free, and keeps working when this service does not answer. This door adds the half that
315
- arithmetic cannot produce: what the ROADS actually cost. Splitting it that way is why the Start
316
- picker and the round-trip toggle keep working with no network at all."""
317
- stops = body.get("stops")
318
- if not isinstance(stops, list) or len(stops) < 2:
319
- raise err(400, "too_few_stops", "A route needs at least two stops with a location.")
320
- if len(stops) > 25:
321
- raise err(400, "too_many_stops",
322
- "Route at most 25 stops at once. Narrow the selection and try again.")
323
- pairs = []
324
- for s in stops:
325
- try:
326
- lat, lon = float(s["lat"]), float(s["lon"])
327
- except (TypeError, ValueError, KeyError, IndexError):
328
- raise err(400, "bad_stop", "Every stop needs a numeric latitude and longitude.")
329
- if not (-90 <= lat <= 90) or not (-180 <= lon <= 180):
330
- raise err(400, "bad_stop", "Every stop needs a latitude and longitude on the globe.")
331
- pairs.append(f"{lon:.6f},{lat:.6f}")
332
-
333
- base = _env("AIOS_ROUTE_URL", _ROUTE_URL_DEFAULT)
334
- _throttle(_env_int("AIOS_GEOCODE_MIN_INTERVAL_MS", _MIN_INTERVAL_MS_DEFAULT))
335
- try:
336
- r = requests.get(
337
- base.rstrip("/") + "/" + ";".join(pairs),
338
- params={"overview": "simplified", "geometries": "geojson", "steps": "false"},
339
- headers={"User-Agent": _env("AIOS_GEO_USER_AGENT", _UA_DEFAULT)},
340
- timeout=20,
341
- )
342
- r.raise_for_status()
343
- data = r.json()
344
- except Exception:
345
- raise err(502, "route_service_unavailable",
346
- "The routing service did not answer. The stop order and the straight line "
347
- "distance are still on the map.")
348
- routes = (data or {}).get("routes") or []
349
- if not routes:
350
- raise err(
351
- 502,
352
- "no_route",
353
- "No road route connects these stops. They may be on different land masses, or one of "
354
- "them may be far from any road.",
355
- )
356
- top = routes[0]
357
- geom = ((top.get("geometry") or {}).get("coordinates")) or []
358
- line = []
359
- for c in geom:
360
- try:
361
- line.append([float(c[0]), float(c[1])])
362
- except (TypeError, ValueError, IndexError):
363
- continue
364
- return {
365
- "km": round(float(top.get("distance") or 0.0) / 1000.0, 1),
366
- "minutes": int(round(float(top.get("duration") or 0.0) / 60.0)),
367
- "line": line,
368
- "attribution": provider_config()["route"]["attribution"],
369
- }
 
1
+ """routes_geo.py — the MAP PROVIDER SEAM (wave 37, R3/R14, tickets T35/T36/T37).
2
+
3
+ ⭐ WHY THIS FILE EXISTS AT ALL, and why the map does not simply call a vendor.
4
+
5
+ R14 (owner, 2026-08-19): no Google Maps API key this wave. The map work is not dropped, it is
6
+ retargeted to providers that need no key: OpenStreetMap tiles, OSRM routing, Nominatim geocoding.
7
+ The deferral only holds if swapping a paid vendor back in is a CONFIG change rather than a rewrite,
8
+ so every provider is a (base URL + attribution) pair read from the environment and served from ONE
9
+ door. The renderer receives a URL template and a credit line; it knows nothing about who is behind
10
+ them. That is the seam.
11
+
12
+ tiles AIOS_MAP_TILE_URL default https://tile.openstreetmap.org/{z}/{x}/{y}.png
13
+ geocode AIOS_GEOCODE_URL default https://nominatim.openstreetmap.org/search
14
+ routing AIOS_ROUTE_URL default https://router.project-osrm.org/route/v1/driving/
15
+
16
+ ⛔ R3'S ON-DEMAND RULE, AND WHY IT IS STRUCTURAL HERE RATHER THAN A COMMENT.
17
+ R3 forbids geocoding a whole table automatically. Under R14 the reason moves from money to manners
18
+ and gets sharper: Nominatim's usage policy allows roughly one request per second and explicitly
19
+ forbids bulk harvesting, so a loop over 3,636 customers does not produce a bill, it produces a
20
+ BLOCKED tenant. Three mechanisms enforce it, none of them advisory:
21
+
22
+ 1. `/geo/geocode` takes an explicit LIST OF ADDRESSES from the caller. It cannot read a table, so
23
+ there is no code path from "a database exists" to "its rows were geocoded". Somebody had to
24
+ choose the records.
25
+ 2. `MAX_BATCH` refuses an oversized list with a named error rather than truncating it. The client
26
+ drives the loop and can therefore SHOW the wait, which is the half of the ticket a comment
27
+ cannot satisfy.
28
+ 3. `_throttle()` blocks in-process until the minimum interval has elapsed, so even a caller that
29
+ ignores everything above cannot exceed the published rate.
30
+
31
+ ⭐ AND A CACHE, because the policy asks for one: an address geocoded once is answered from memory
32
+ for the rest of the container's life. It is the cheapest way to be a good citizen and it makes a
33
+ re-run of the same records free rather than merely legal.
34
+
35
+ ⚠ NOTHING HERE PROXIES TILES. Tiles are fetched by the BROWSER, straight from the provider, which
36
+ is what every OSM client does and what the tile policy expects. Proxying them through this app
37
+ would put a free tier in the path of every pan and would breach the same policy it looks like it is
38
+ respecting. This door serves the tile URL, never the tile.
39
+ """
40
+ import os
41
+ import re
42
+ import threading
43
+ import time
44
+ from collections import OrderedDict
45
+
46
+ import requests
47
+ from fastapi import APIRouter, Body, Depends
48
+
49
+ from deps import Session, err, require_session
50
+
51
+ router = APIRouter(prefix="/api/v1")
52
+
53
+ # --------------------------------------------------------------------- the seam
54
+ #
55
+ # Every value below is an environment override with a keyless default. Pointing the map at a paid
56
+ # vendor is three variables on the Space and no code change, which is what makes R14 a deferral.
57
+
58
+ _TILE_URL_DEFAULT = "https://tile.openstreetmap.org/{z}/{x}/{y}.png"
59
+ _TILE_CREDIT_DEFAULT = "© OpenStreetMap contributors"
60
+ _TILE_CREDIT_HREF_DEFAULT = "https://www.openstreetmap.org/copyright"
61
+ _GEOCODE_URL_DEFAULT = "https://nominatim.openstreetmap.org/search"
62
+ _ROUTE_URL_DEFAULT = "https://router.project-osrm.org/route/v1/driving/"
63
+
64
+ #: Both policies require a real, identifying User-Agent. A generic one is how a shared service
65
+ #: decides an application is a scraper, so this carries the product name and a contact URL.
66
+ _UA_DEFAULT = "AIOS-Loopable/1.0 (+https://runloopable.com)"
67
+
68
+ #: Nominatim publishes one request per second. 1100 ms leaves room for clock jitter rather than
69
+ #: sitting exactly on the published edge.
70
+ _MIN_INTERVAL_MS_DEFAULT = 1100
71
+
72
+ #: The most addresses one call may carry. Small on purpose: the client loops and shows progress,
73
+ #: and no single request can sit on the connection for a minute waiting out the throttle.
74
+ MAX_BATCH = 25
75
+
76
+ #: Tile zoom the provider actually serves. OSM stops at 19.
77
+ _TILE_MAX_Z_DEFAULT = 19
78
+
79
+ _CACHE_MAX = 4096
80
+
81
+
82
+ def _env(name, default):
83
+ return (os.environ.get(name) or "").strip() or default
84
+
85
+
86
+ def _env_int(name, default):
87
+ try:
88
+ n = int((os.environ.get(name) or "").strip())
89
+ return n if n > 0 else default
90
+ except (TypeError, ValueError):
91
+ return default
92
+
93
+
94
+ def provider_config():
95
+ """The whole seam as one dict. `GET /geo/providers` serves it and the gate reads it."""
96
+ return {
97
+ "tiles": {
98
+ "url": _env("AIOS_MAP_TILE_URL", _TILE_URL_DEFAULT),
99
+ "attribution": _env("AIOS_MAP_TILE_ATTRIBUTION", _TILE_CREDIT_DEFAULT),
100
+ "attributionUrl": _env("AIOS_MAP_TILE_ATTRIBUTION_URL", _TILE_CREDIT_HREF_DEFAULT),
101
+ "maxZoom": _env_int("AIOS_MAP_TILE_MAX_Z", _TILE_MAX_Z_DEFAULT),
102
+ },
103
+ "geocode": {
104
+ "available": True,
105
+ "maxBatch": MAX_BATCH,
106
+ "minIntervalMs": _env_int("AIOS_GEOCODE_MIN_INTERVAL_MS", _MIN_INTERVAL_MS_DEFAULT),
107
+ # ⭐⭐ THE NUMBER THE SURFACE MUST QUOTE, AND IT IS NOT THE RATE LIMIT.
108
+ # T37 asks for the estimated wait to be shown BEFORE a run. The obvious source is
109
+ # `minIntervalMs`, and it is wrong: 1.1 s is a floor on POLITENESS, not a prediction of
110
+ # LATENCY. One real lookup was MEASURED at 3.28 s end to end on 2026-08-19. Quoting the
111
+ # limit would promise a minute and take three, and a progress estimate that runs out
112
+ # before the work does reads as a hang rather than as a wait.
113
+ "secondsPerAddress": float(_env("AIOS_GEOCODE_SECONDS_EACH", "3.3")),
114
+ "attribution": _env("AIOS_GEOCODE_ATTRIBUTION", _TILE_CREDIT_DEFAULT),
115
+ },
116
+ "route": {
117
+ "available": True,
118
+ "attribution": _env("AIOS_ROUTE_ATTRIBUTION", "Routing by OSRM"),
119
+ },
120
+ }
121
+
122
+
123
+ # ------------------------------------------------------------------ the throttle
124
+ #
125
+ # ⚠ ONE lock and ONE timestamp for the whole process, shared by geocoding and routing, because the
126
+ # rate limit belongs to the SERVICE and not to the endpoint. Two independent throttles would each
127
+ # stay honest and together double the published rate.
128
+
129
+ _gate = threading.Lock()
130
+ _last_call = [0.0]
131
+
132
+
133
+ def _throttle(min_interval_ms):
134
+ """Block until the minimum interval since the last outbound call has elapsed.
135
+
136
+ Returns the seconds actually waited, so a caller can report the wait rather than hide it."""
137
+ wait = 0.0
138
+ with _gate:
139
+ gap = min_interval_ms / 1000.0
140
+ now = time.monotonic()
141
+ due = _last_call[0] + gap
142
+ if now < due:
143
+ wait = due - now
144
+ time.sleep(wait)
145
+ _last_call[0] = time.monotonic()
146
+ return wait
147
+
148
+
149
+ # --------------------------------------------------------------------- the cache
150
+
151
+ _cache = OrderedDict()
152
+ _cache_lock = threading.Lock()
153
+
154
+
155
+ def _cache_key(q, country):
156
+ return (re.sub(r"\s+", " ", str(q or "")).strip().lower(), str(country or "").strip().lower())
157
+
158
+
159
+ def _cache_get(key):
160
+ with _cache_lock:
161
+ if key not in _cache:
162
+ return None
163
+ _cache.move_to_end(key)
164
+ return _cache[key]
165
+
166
+
167
+ def _cache_put(key, value):
168
+ with _cache_lock:
169
+ _cache[key] = value
170
+ _cache.move_to_end(key)
171
+ while len(_cache) > _CACHE_MAX:
172
+ _cache.popitem(last=False)
173
+
174
+
175
+ def geocode_one(address, country=None, timeout=12):
176
+ """One address to {lat, lon, label} or None.
177
+
178
+ ⛔ THE ONLY PLACE AN OUTBOUND GEOCODE HAPPENS. `ai_enrich.py`'s geocode field calls this rather
179
+ than reaching for `requests` itself, so the throttle and the cache cannot be walked around by a
180
+ second caller. A cache hit costs no request and no wait."""
181
+ q = re.sub(r"\s+", " ", str(address or "")).strip()
182
+ if not q:
183
+ return None
184
+ key = _cache_key(q, country)
185
+ hit = _cache_get(key)
186
+ if hit is not None:
187
+ return dict(hit) if hit else None
188
+
189
+ cfg = provider_config()["geocode"]
190
+ _throttle(cfg["minIntervalMs"])
191
+ params = {"q": q, "format": "jsonv2", "limit": 1}
192
+ if country:
193
+ params["countrycodes"] = str(country).strip().lower()
194
+ try:
195
+ r = requests.get(
196
+ _env("AIOS_GEOCODE_URL", _GEOCODE_URL_DEFAULT),
197
+ params=params,
198
+ headers={"User-Agent": _env("AIOS_GEO_USER_AGENT", _UA_DEFAULT),
199
+ "Accept": "application/json"},
200
+ timeout=timeout,
201
+ )
202
+ r.raise_for_status()
203
+ rows = r.json()
204
+ except Exception:
205
+ # ⚠ A transport failure is NOT cached. Caching it would turn one bad minute into a
206
+ # permanently empty column that no re-run could ever repair.
207
+ return None
208
+ if not isinstance(rows, list) or not rows:
209
+ # A genuine "no such place" IS cached, as a negative: asking again gets the same answer and
210
+ # spends another second of a shared service's budget to hear it.
211
+ _cache_put(key, {})
212
+ return None
213
+ top = rows[0] or {}
214
+ try:
215
+ lat = float(top.get("lat"))
216
+ lon = float(top.get("lon"))
217
+ except (TypeError, ValueError):
218
+ _cache_put(key, {})
219
+ return None
220
+ if not (-90 <= lat <= 90) or not (-180 <= lon <= 180):
221
+ _cache_put(key, {})
222
+ return None
223
+ out = {"lat": lat, "lon": lon, "label": str(top.get("display_name") or q)}
224
+ _cache_put(key, out)
225
+ return dict(out)
226
+
227
+
228
+ # ---------------------------------------------------------------------- the doors
229
+
230
+
231
+ @router.get("/geo/providers")
232
+ def geo_providers(session: Session = Depends(require_session)):
233
+ """What the map should draw with, and who to credit for it.
234
+
235
+ Session gated like every other door here. The values are not secret, but an unauthenticated
236
+ endpoint on this app is a door somebody eventually hangs something else on."""
237
+ return provider_config()
238
+
239
+
240
+ @router.post("/geo/geocode")
241
+ def geo_geocode(body: dict = Body(...), session: Session = Depends(require_session)):
242
+ """Turn a bounded list of addresses into coordinates.
243
+
244
+ ⛔ Takes ADDRESSES, never a table key, a view id or a filter. There is deliberately no shape of
245
+ request that means "geocode everything", which is R3 expressed as an API rather than as a
246
+ warning."""
247
+ items = body.get("addresses")
248
+ # ⭐ THE SECOND SHAPE, AND IT EXISTS TO KEEP ONE COMPOSER. A caller may send whole RECORDS plus
249
+ # the geocode field's config instead of finished address strings, and the server builds the
250
+ # query with `ai_enrich.geocode_query`. The alternative was a TypeScript twin of that function
251
+ # on the client, and an address composer that exists twice will disagree the first time either
252
+ # copy learns about a new column ([[one-question-two-normalizers]]) -- which is exactly the trap
253
+ # B's measured `New York (US)` suffix would spring, silently, in whichever copy forgot.
254
+ if items is None:
255
+ records = body.get("records")
256
+ cfg = body.get("config") or {}
257
+ if isinstance(records, list) and records:
258
+ import ai_enrich
259
+ items = []
260
+ for rec in records:
261
+ if not isinstance(rec, dict):
262
+ continue
263
+ q = ai_enrich.geocode_query(rec.get("row") or {}, cfg)
264
+ # ⛔ A record with no usable address is REPORTED, not dropped: it comes back
265
+ # `found: false` with an empty address, so the caller can name it on screen instead
266
+ # of quietly returning fewer answers than it asked questions.
267
+ items.append({"key": rec.get("key"), "address": q})
268
+ if not body.get("country") and cfg.get("country"):
269
+ body = {**body, "country": cfg.get("country")}
270
+ if not isinstance(items, list) or not items:
271
+ raise err(400, "no_addresses", "Send at least one address to look up.")
272
+ if len(items) > MAX_BATCH:
273
+ raise err(
274
+ 400,
275
+ "batch_too_large",
276
+ f"Look up at most {MAX_BATCH} addresses per request. "
277
+ f"The map sends them in batches of {MAX_BATCH} so the wait stays visible.",
278
+ )
279
+ country = body.get("country")
280
+ out = []
281
+ started = time.monotonic()
282
+ for raw in items:
283
+ if isinstance(raw, dict):
284
+ key, addr = raw.get("key"), raw.get("address")
285
+ else:
286
+ key, addr = None, raw
287
+ # ⛔ AN EMPTY QUERY COSTS NO REQUEST AND CARRIES ITS OWN REASON. Nominatim's policy is a
288
+ # budget shared with everyone else using it, and asking it to place "" would spend a second
289
+ # of that budget to be told nothing. `reason` is what lets the surface say "no address on
290
+ # file" rather than "not found", which are different facts and want different actions.
291
+ if not str(addr or "").strip():
292
+ out.append({"key": key, "address": "", "found": False, "lat": None, "lon": None,
293
+ "label": None, "reason": "no_address"})
294
+ continue
295
+ hit = geocode_one(addr, country=country)
296
+ out.append({"key": key, "address": str(addr or ""), "found": bool(hit),
297
+ "reason": None if hit else "not_found",
298
+ "lat": hit["lat"] if hit else None,
299
+ "lon": hit["lon"] if hit else None,
300
+ "label": hit["label"] if hit else None})
301
+ elapsed = time.monotonic() - started
302
+ return {"results": out,
303
+ "found": sum(1 for r in out if r["found"]),
304
+ "asked": len(out),
305
+ "secondsElapsed": round(elapsed, 2),
306
+ "attribution": provider_config()["geocode"]["attribution"]}
307
+
308
+
309
+ @router.post("/geo/route")
310
+ def geo_route(body: dict = Body(...), session: Session = Depends(require_session)):
311
+ """Road distance, duration and the drawn line, for stops ALREADY put in order.
312
+
313
+ ⭐ THE ORDERING IS NOT DONE HERE. `mapProjection.planRoute` sequences the stops on the client,
314
+ for free, and keeps working when this service does not answer. This door adds the half that
315
+ arithmetic cannot produce: what the ROADS actually cost. Splitting it that way is why the Start
316
+ picker and the round-trip toggle keep working with no network at all."""
317
+ stops = body.get("stops")
318
+ if not isinstance(stops, list) or len(stops) < 2:
319
+ raise err(400, "too_few_stops", "A route needs at least two stops with a location.")
320
+ if len(stops) > 25:
321
+ raise err(400, "too_many_stops",
322
+ "Route at most 25 stops at once. Narrow the selection and try again.")
323
+ pairs = []
324
+ for s in stops:
325
+ try:
326
+ lat, lon = float(s["lat"]), float(s["lon"])
327
+ except (TypeError, ValueError, KeyError, IndexError):
328
+ raise err(400, "bad_stop", "Every stop needs a numeric latitude and longitude.")
329
+ if not (-90 <= lat <= 90) or not (-180 <= lon <= 180):
330
+ raise err(400, "bad_stop", "Every stop needs a latitude and longitude on the globe.")
331
+ pairs.append(f"{lon:.6f},{lat:.6f}")
332
+
333
+ base = _env("AIOS_ROUTE_URL", _ROUTE_URL_DEFAULT)
334
+ _throttle(_env_int("AIOS_GEOCODE_MIN_INTERVAL_MS", _MIN_INTERVAL_MS_DEFAULT))
335
+ try:
336
+ r = requests.get(
337
+ base.rstrip("/") + "/" + ";".join(pairs),
338
+ params={"overview": "simplified", "geometries": "geojson", "steps": "false"},
339
+ headers={"User-Agent": _env("AIOS_GEO_USER_AGENT", _UA_DEFAULT)},
340
+ timeout=20,
341
+ )
342
+ r.raise_for_status()
343
+ data = r.json()
344
+ except Exception:
345
+ raise err(502, "route_service_unavailable",
346
+ "The routing service did not answer. The stop order and the straight line "
347
+ "distance are still on the map.")
348
+ routes = (data or {}).get("routes") or []
349
+ if not routes:
350
+ raise err(
351
+ 502,
352
+ "no_route",
353
+ "No road route connects these stops. They may be on different land masses, or one of "
354
+ "them may be far from any road.",
355
+ )
356
+ top = routes[0]
357
+ geom = ((top.get("geometry") or {}).get("coordinates")) or []
358
+ line = []
359
+ for c in geom:
360
+ try:
361
+ line.append([float(c[0]), float(c[1])])
362
+ except (TypeError, ValueError, IndexError):
363
+ continue
364
+ return {
365
+ "km": round(float(top.get("distance") or 0.0) / 1000.0, 1),
366
+ "minutes": int(round(float(top.get("duration") or 0.0) / 60.0)),
367
+ "line": line,
368
+ "attribution": provider_config()["route"]["attribution"],
369
+ }
api/routes_grid.py CHANGED
The diff for this file is too large to render. See raw diff
 
api/routes_keychain.py CHANGED
The diff for this file is too large to render. See raw diff
 
api/routes_nav.py CHANGED
The diff for this file is too large to render. See raw diff
 
api/routes_oauth.py CHANGED
@@ -1,114 +1,114 @@
1
- """routes_oauth.py — the OAuth connector surface (wave 22, contract C5 + A2/A3 / R12).
2
-
3
- Thin over `oauth_connect`, the way `routes_automation` is thin over the engine: sessions,
4
- shapes and status codes here; every decision that could be wrong lives in the module a gate
5
- can drive without a server. GENERIC over `{provider}` (C5-A2): the routes read the registry,
6
- so the day a second provider lands here is the day nothing in this file changes.
7
-
8
- MOUNTED FROM `routes_automation` (not `main.py`): this wave's ownership fence gives no session
9
- `main.py`, and `routes_automation` is already included there — so this router rides inside it
10
- (`/api/v1` + `/oauth/...`). Lifting the include into `main.py` later is a two-line change that
11
- alters no path.
12
-
13
- ⚠ THE TWO REDIRECT LAWS (A3): `/{provider}/start` answers **302 to the provider's consent
14
- screen** — it is a top-level navigation the client reaches by `<a href>`, never JSON. The
15
- callback 302s BACK to the return path the `state` carried (relative-only, sanitised by
16
- `oauth_connect.safe_next`), so the user lands where they left — connected or not, whatever
17
- went wrong rides in the query string; a dead-end error page where the app used to be reads as
18
- "the product broke", not "the connect failed".
19
- """
20
- import os
21
-
22
- from fastapi import APIRouter, Depends, Request
23
- from fastapi.responses import RedirectResponse
24
-
25
- import oauth_connect
26
- from deps import Session, err, require_session
27
-
28
- router = APIRouter(prefix="/oauth")
29
-
30
-
31
- def _redirect_uri(request: Request, provider: str) -> str:
32
- """The redirect URI this deployment registers at the provider — env-pinned when the
33
- container sits behind a proxy that rewrites the scheme (the HF Space), else derived from
34
- the request. MUST match a console-registered URI verbatim, so it is computed in exactly
35
- one place.
36
-
37
- ⭐ WAVE 29 (R4): `deploy_web.py` now PUSHES `AIOS_PUBLIC_BASE` on every deploy, defaulted to
38
- the same URL as `APP_BASE_URL`, so the pinned branch is the one that runs in production and
39
- the request-derived fallback below is effectively dev-only.
40
- ⛔ THAT MAKES THIS FUNCTION A CUSTOM-DOMAIN COUPLING, not merely a scheme fix. Whatever host
41
- this returns is where the provider sends the user BACK, and the session cookie is host-only
42
- (`aios_session.py:114-117`, no `domain=`) — so a callback base that disagrees with the host
43
- the user actually browsed plants the session on the wrong hostname and they return logged
44
- out. Moving the app to a new hostname means moving this value AND re-registering the
45
- resulting URI in the provider console; one without the other fails closed.
46
- Runbook: `.claude/wiki/research/loopable-domain-runbook.md`."""
47
- base = (os.environ.get("AIOS_PUBLIC_BASE") or "").strip().rstrip("/")
48
- if not base:
49
- base = f"{request.url.scheme}://{request.url.netloc}"
50
- return f"{base}/api/v1/oauth/{provider}/callback"
51
-
52
-
53
- @router.get("/status")
54
- def oauth_status(session: Session = Depends(require_session)):
55
- """C5's status shape for the SESSION user, one entry per registry provider:
56
- `{google: {connected, email, reconnect, configured}}` today. The bit the email trigger's
57
- `ready` reads through."""
58
- return oauth_connect.status(session.runtime, session.uname)
59
-
60
-
61
- def _offered_or_404(provider: str):
62
- """⛔⛔ W32-T14 / OWNER ITEM 12 / R6 — A PROVIDER THE PRODUCT DOES NOT OFFER HAS NO DOOR.
63
-
64
- The owner pasted the failure this replaces: clicking Connect on the Google card answered
65
- `503 oauth_unavailable` as raw JSON. R6's fix is not a nicer error — it is that the flow is
66
- not offered at all, so the honest status is the one for a URL that does not exist. `404`
67
- rather than `503`, deliberately: a 503 says *"come back later"* about a door that is not
68
- coming back until somebody pays for CASA verification (D-45, ~$540–1,800/yr).
69
-
70
- ⚠ Every door in this router goes through it, START included, because the JSON the owner saw
71
- came from the start route and a guard on one leg is a guard on one leg.
72
- """
73
- if not oauth_connect.offered(provider):
74
- raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider")
75
-
76
-
77
- @router.get("/{provider}/start")
78
- def oauth_start(provider: str, request: Request, next: str = "",
79
- session: Session = Depends(require_session)):
80
- """302 to the provider's consent screen (A3 — a navigation, never JSON). `?next=` is the
81
- RELATIVE path the callback returns the browser to; it rides inside the single-use state,
82
- sanitised, so the round trip cannot be steered off-origin."""
83
- _offered_or_404(provider)
84
- url, problem = oauth_connect.start(provider, session.uname,
85
- _redirect_uri(request, provider), next_path=next)
86
- if problem:
87
- raise err(503 if "not configured" in problem else 404, "oauth_unavailable", problem)
88
- return RedirectResponse(url, status_code=302)
89
-
90
-
91
- @router.get("/{provider}/callback")
92
- def oauth_callback(provider: str, request: Request,
93
- session: Session = Depends(require_session),
94
- state: str = "", code: str = "", error: str = ""):
95
- """The provider's redirect target. Exchanges the code, stores the per-user slot, and sends
96
- the browser back to the state's return path — connected or not (see module header)."""
97
- _offered_or_404(provider)
98
- if error:
99
- home = "/#/"
100
- return RedirectResponse(f"{home}?oauthError={error[:80]}", status_code=302)
101
- email, home, problem = oauth_connect.callback(session.runtime, session.uname, state, code)
102
- sep = "&" if "?" in home else "?"
103
- if problem:
104
- return RedirectResponse(f"{home}{sep}oauthError=connect_failed", status_code=302)
105
- return RedirectResponse(f"{home}{sep}connected={provider}", status_code=302)
106
-
107
-
108
- @router.post("/{provider}/disconnect")
109
- def oauth_disconnect(provider: str, session: Session = Depends(require_session)):
110
- _offered_or_404(provider)
111
- if oauth_connect.provider_def(provider) is None:
112
- raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider")
113
- oauth_connect.disconnect(session.runtime, session.uname, provider)
114
- return {"disconnected": provider}
 
1
+ """routes_oauth.py — the OAuth connector surface (wave 22, contract C5 + A2/A3 / R12).
2
+
3
+ Thin over `oauth_connect`, the way `routes_automation` is thin over the engine: sessions,
4
+ shapes and status codes here; every decision that could be wrong lives in the module a gate
5
+ can drive without a server. GENERIC over `{provider}` (C5-A2): the routes read the registry,
6
+ so the day a second provider lands here is the day nothing in this file changes.
7
+
8
+ MOUNTED FROM `routes_automation` (not `main.py`): this wave's ownership fence gives no session
9
+ `main.py`, and `routes_automation` is already included there — so this router rides inside it
10
+ (`/api/v1` + `/oauth/...`). Lifting the include into `main.py` later is a two-line change that
11
+ alters no path.
12
+
13
+ ⚠ THE TWO REDIRECT LAWS (A3): `/{provider}/start` answers **302 to the provider's consent
14
+ screen** — it is a top-level navigation the client reaches by `<a href>`, never JSON. The
15
+ callback 302s BACK to the return path the `state` carried (relative-only, sanitised by
16
+ `oauth_connect.safe_next`), so the user lands where they left — connected or not, whatever
17
+ went wrong rides in the query string; a dead-end error page where the app used to be reads as
18
+ "the product broke", not "the connect failed".
19
+ """
20
+ import os
21
+
22
+ from fastapi import APIRouter, Depends, Request
23
+ from fastapi.responses import RedirectResponse
24
+
25
+ import oauth_connect
26
+ from deps import Session, err, require_session
27
+
28
+ router = APIRouter(prefix="/oauth")
29
+
30
+
31
+ def _redirect_uri(request: Request, provider: str) -> str:
32
+ """The redirect URI this deployment registers at the provider — env-pinned when the
33
+ container sits behind a proxy that rewrites the scheme (the HF Space), else derived from
34
+ the request. MUST match a console-registered URI verbatim, so it is computed in exactly
35
+ one place.
36
+
37
+ ⭐ WAVE 29 (R4): `deploy_web.py` now PUSHES `AIOS_PUBLIC_BASE` on every deploy, defaulted to
38
+ the same URL as `APP_BASE_URL`, so the pinned branch is the one that runs in production and
39
+ the request-derived fallback below is effectively dev-only.
40
+ ⛔ THAT MAKES THIS FUNCTION A CUSTOM-DOMAIN COUPLING, not merely a scheme fix. Whatever host
41
+ this returns is where the provider sends the user BACK, and the session cookie is host-only
42
+ (`aios_session.py:114-117`, no `domain=`) — so a callback base that disagrees with the host
43
+ the user actually browsed plants the session on the wrong hostname and they return logged
44
+ out. Moving the app to a new hostname means moving this value AND re-registering the
45
+ resulting URI in the provider console; one without the other fails closed.
46
+ Runbook: `.claude/wiki/research/loopable-domain-runbook.md`."""
47
+ base = (os.environ.get("AIOS_PUBLIC_BASE") or "").strip().rstrip("/")
48
+ if not base:
49
+ base = f"{request.url.scheme}://{request.url.netloc}"
50
+ return f"{base}/api/v1/oauth/{provider}/callback"
51
+
52
+
53
+ @router.get("/status")
54
+ def oauth_status(session: Session = Depends(require_session)):
55
+ """C5's status shape for the SESSION user, one entry per registry provider:
56
+ `{google: {connected, email, reconnect, configured}}` today. The bit the email trigger's
57
+ `ready` reads through."""
58
+ return oauth_connect.status(session.runtime, session.uname)
59
+
60
+
61
+ def _offered_or_404(provider: str):
62
+ """⛔⛔ W32-T14 / OWNER ITEM 12 / R6 — A PROVIDER THE PRODUCT DOES NOT OFFER HAS NO DOOR.
63
+
64
+ The owner pasted the failure this replaces: clicking Connect on the Google card answered
65
+ `503 oauth_unavailable` as raw JSON. R6's fix is not a nicer error — it is that the flow is
66
+ not offered at all, so the honest status is the one for a URL that does not exist. `404`
67
+ rather than `503`, deliberately: a 503 says *"come back later"* about a door that is not
68
+ coming back until somebody pays for CASA verification (D-45, ~$540–1,800/yr).
69
+
70
+ ⚠ Every door in this router goes through it, START included, because the JSON the owner saw
71
+ came from the start route and a guard on one leg is a guard on one leg.
72
+ """
73
+ if not oauth_connect.offered(provider):
74
+ raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider")
75
+
76
+
77
+ @router.get("/{provider}/start")
78
+ def oauth_start(provider: str, request: Request, next: str = "",
79
+ session: Session = Depends(require_session)):
80
+ """302 to the provider's consent screen (A3 — a navigation, never JSON). `?next=` is the
81
+ RELATIVE path the callback returns the browser to; it rides inside the single-use state,
82
+ sanitised, so the round trip cannot be steered off-origin."""
83
+ _offered_or_404(provider)
84
+ url, problem = oauth_connect.start(provider, session.uname,
85
+ _redirect_uri(request, provider), next_path=next)
86
+ if problem:
87
+ raise err(503 if "not configured" in problem else 404, "oauth_unavailable", problem)
88
+ return RedirectResponse(url, status_code=302)
89
+
90
+
91
+ @router.get("/{provider}/callback")
92
+ def oauth_callback(provider: str, request: Request,
93
+ session: Session = Depends(require_session),
94
+ state: str = "", code: str = "", error: str = ""):
95
+ """The provider's redirect target. Exchanges the code, stores the per-user slot, and sends
96
+ the browser back to the state's return path — connected or not (see module header)."""
97
+ _offered_or_404(provider)
98
+ if error:
99
+ home = "/#/"
100
+ return RedirectResponse(f"{home}?oauthError={error[:80]}", status_code=302)
101
+ email, home, problem = oauth_connect.callback(session.runtime, session.uname, state, code)
102
+ sep = "&" if "?" in home else "?"
103
+ if problem:
104
+ return RedirectResponse(f"{home}{sep}oauthError=connect_failed", status_code=302)
105
+ return RedirectResponse(f"{home}{sep}connected={provider}", status_code=302)
106
+
107
+
108
+ @router.post("/{provider}/disconnect")
109
+ def oauth_disconnect(provider: str, session: Session = Depends(require_session)):
110
+ _offered_or_404(provider)
111
+ if oauth_connect.provider_def(provider) is None:
112
+ raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider")
113
+ oauth_connect.disconnect(session.runtime, session.uname, provider)
114
+ return {"disconnected": provider}
api/routes_publish.py CHANGED
The diff for this file is too large to render. See raw diff
 
api/routes_records.py CHANGED
@@ -1,211 +1,211 @@
1
- """Record detail routes: durable comments, scoped to the caller's book — ON EVERY DATABASE.
2
-
3
- ⭐ WAVE 19 (owner item 12). This file used to be the CUSTOMER record's comment routes with a
4
- customer-shaped wall bolted to the module import line: `_in_book` asked
5
- `routes_customers.allowed_pids` whatever surface the browser was on. Opening a PRODUCT record and
6
- typing a comment therefore asked the customer book about a CRC32 hash of a SKU code, and the
7
- panel answered "that customer is not in your book" — the owner's report. The dangerous half is
8
- the one nobody sees: a hash that collides with a real partner id passes the wall, and the comment
9
- is filed against somebody's customer where the whole team can read it.
10
-
11
- THE SHAPE NOW: `?scope=` names the database (the same vocabulary `/workspace?scope=` and the
12
- events route's `scopeKey` already speak), and `_pool_or_refuse` resolves BOTH halves of the wall
13
- per scope — the GRANT and the ROW SET — by asking that topic's own route, never by re-deriving
14
- one here:
15
-
16
- customer / cohort `routes_customers.allowed_pids` behind the `customer_data` grant
17
- product `routes_products.scoped_pool` behind the `product_data` grant
18
- ut_<slug> `routes_tables.scoped_pids`, whose `_defn_or_refuse` IS the wall
19
- (404 unknown / 403 not yours — a user table has no module grant).
20
- ⭐ W33-T03/D-183: `scoped_pIDs`, not `scoped_pOOL` — the pool builds every
21
- ROW to derive a pid set this module discards, and RAISES 409 on a
22
- read-through grid past one window, which is why the record drawer painted
23
- an error page on `ut_odoo_gl_lines`.
24
-
25
- ⚠ THE PATH KEEPS ITS `/customers/` SEGMENT. It is the shipped URL and `verify_api.py`'s E1a
26
- section pins it; the scope now travels beside it explicitly. A nicer noun is not worth churning
27
- another session's gate mid-wave — the WALL is the query parameter, not the word.
28
-
29
- ⚠ NO DEFAULT BEYOND THE LEGACY ONE. An absent `scope` means `customer`, which is what every
30
- shipped client sent and what keeps the old callers byte-identical; an UNRECOGNISED scope is a
31
- 400, never a silent fallback to the customer book (`routes_grid._scope_or_400`'s rule, and for
32
- the same reason: a typo served as `customer` answers a question nobody asked).
33
- """
34
- from fastapi import APIRouter, Body, Depends, Query
35
-
36
- from deps import Session, err, require_session
37
-
38
- router = APIRouter(prefix="/api/v1")
39
-
40
- #: The customer topic's two names — one book, two surfaces (the Cohort page is the customer table
41
- #: over hand-curated sets). Mirrors `modules.cohort.LEGACY_SCOPES` / `core.record_comments`.
42
- _CUSTOMER_SCOPES = ("", "customer", "cohort")
43
-
44
-
45
- def _scope_or_400(raw):
46
- scope = str(raw or "customer").strip().lower()
47
- if scope in _CUSTOMER_SCOPES or scope == "product" or scope.startswith("ut_"):
48
- return "customer" if scope in _CUSTOMER_SCOPES else scope
49
- raise err(400, "bad_scope",
50
- "scope must be customer, cohort, product or a ut_ database — refusing to guess")
51
-
52
-
53
- def _pool_or_refuse(session: Session, scope: str):
54
- """The pids this session may attach comments to ON THIS DATABASE — grant wall included.
55
-
56
- Returns **`(pids, unbounded)`** — a 2-tuple on EVERY branch. Raises the topic's own 403/404/503,
57
- so a caller who may not open the surface never learns anything about the row they asked about.
58
-
59
- ⛔ `unbounded` is TRUE only when the row set could not be ENUMERATED (a read-through grid past
60
- one window), never when it is merely EMPTY. Those are opposite answers and `scoped_pids` returns
61
- `frozenset()` for both — see the branch below.
62
- ⚠ THE SHAPE IS A CONTRACT EVEN THOUGH THIS FUNCTION IS PRIVATE, and it has two consumers that
63
- do not travel together: `_in_book` here, and a NEGATIVE CONTROL in `aios-web/api/verify_scopes.py`
64
- that REPLACES this function with its own lambda. A gate's test double is a caller
65
- ([[test-double-patched-by-a-name-list]]); when this signature moved, that double kept returning a
66
- bare frozenset and the section died on `ValueError: too many values to unpack` — no tally, no
67
- failing name. Change the shape here and that double changes with it.
68
- """
69
- if scope == "product":
70
- from routes_products import MODULE as PRODUCT_MODULE, scoped_pool
71
-
72
- session.require(PRODUCT_MODULE)
73
- pids, _team, _rows, _fields = scoped_pool(session)
74
- # ⚠ `(pids, unbounded)` on EVERY branch. This one returned a bare frozenset for ten minutes
75
- # after the `ut_` branch grew its second element, and `_in_book`'s unpack would have raised
76
- # a `TypeError` — a 500 on every product comment — while both other branches worked. A
77
- # return shape is a contract even when the function is private.
78
- return (pids, False)
79
- if scope.startswith("ut_"):
80
- # No module grant exists for a user table — `_defn_or_refuse` inside `scoped_pids` IS
81
- # the wall (creator or admin, fail-closed), and it answers 404 before 403 exactly as the
82
- # rows routes do.
83
- #
84
- # ⭐⭐ W33-T03 / D-183 — `scoped_pids`, NOT `scoped_pool`, AND THAT ONE WORD IS THE BUG.
85
- # `scoped_pool` builds every ROW to derive a pid set this function then throws away, and on
86
- # a read-through grid larger than one window it RAISES `409 window_required`. So opening the
87
- # record drawer on `ut_odoo_gl_lines` (975,137 rows) painted an error page — for a panel
88
- # that renders comments about ONE row it already has. `scoped_pids` answers the identical
89
- # question (its docstring: *"the pid set is IDENTICAL, not merely equivalent"*) and takes
90
- # W31-T20's `limits` OUT-PARAMETER instead of raising, which is the same shape `/workspace`
91
- # used to become openable on those two grids.
92
- from routes_tables import scoped_pids
93
-
94
- limits = []
95
- pids, _fields, _defn = scoped_pids(session, scope, limits=limits)
96
- # ⛔ AN EMPTY PID SET AND AN UNRESOLVABLE ONE ARE OPPOSITE ANSWERS, and collapsing them is
97
- # how a fail-closed default becomes a lie. `scoped_pids` returns `frozenset()` BOTH for a
98
- # database with no rows and for a read-through grid too big to enumerate — it distinguishes
99
- # them by APPENDING R6's sentence to `limits`. Without this branch the drawer would move
100
- # from a 409 error page to a 403 "not in your book" on a row the user is looking at, which
101
- # is the same defect wearing a politer message ([[empty-answer-vs-unfinished-answer]]).
102
- # ⚠ ADMITTING HERE IS NOT A WIDENING, and the ruling is wave 27 / D-72: on a `ut_*` database
103
- # THE TENANT IS THE UNIT — `scoped_pool` itself carries "no per-row owner filter; the
104
- # table-level wall is the WHOLE wall". `_defn_or_refuse` has already run inside
105
- # `scoped_pids` and answered 404/403. The pid set was only ever an existence check.
106
- return (pids, bool(limits))
107
- from routes_customers import MODULE as CUSTOMER_MODULE, allowed_pids
108
-
109
- session.require(CUSTOMER_MODULE)
110
- return (frozenset(allowed_pids(session)), False)
111
-
112
-
113
- def _in_book(pid, session, scope):
114
- pids, unbounded = _pool_or_refuse(session, scope)
115
- if unbounded:
116
- # The table wall passed and the row set is larger than this process will enumerate. Said
117
- # out loud rather than silently admitting: R6's second sentence is that a limit which
118
- # cannot be removed gets REPORTED, and this is the one place the report has no envelope to
119
- # ride in.
120
- print(f"[records] {scope}: pid membership unresolved (read-through beyond one window) — "
121
- f"admitting on the table wall alone, per D-72")
122
- return
123
- if pid not in pids:
124
- # 403, not 404: the record may exist, but this session may not inspect it.
125
- raise err(403, "out_of_scope", "that record is not in your book")
126
-
127
-
128
- def _unavailable():
129
- return err(
130
- 503,
131
- "store_unavailable",
132
- "record comments are temporarily unavailable — no change was saved",
133
- )
134
-
135
-
136
- # ⭐ WAVE 21 (D-17): the CANONICAL path is /records/{pid}/comments — comments hang off a RECORD
137
- # in whatever topic `?scope=` names, and the customer-flavoured noun was wave-19 residue (the
138
- # wall was always the query param). The old path stays as an ALIAS because the shipped client
139
- # still calls it; verify_api pins the canonical path AND that the alias answers, so removing
140
- # the alias later is a decision, never an accident.
141
- @router.get("/records/{pid}/comments")
142
- @router.get("/customers/{pid}/comments")
143
- def comments(pid: int, scope: str = Query(default="customer"),
144
- session: Session = Depends(require_session)):
145
- from core import record_comments
146
-
147
- scope = _scope_or_400(scope)
148
- _in_book(pid, session, scope)
149
- try:
150
- rows = record_comments.list_comments(session.runtime, pid, scope=scope)
151
- except record_comments.CommentsUnavailable:
152
- raise _unavailable()
153
- return {"comments": rows}
154
-
155
-
156
- @router.post("/records/{pid}/comments", status_code=201)
157
- @router.post("/customers/{pid}/comments", status_code=201)
158
- def create_comment(
159
- pid: int,
160
- body: dict = Body(default=None),
161
- scope: str = Query(default="customer"),
162
- session: Session = Depends(require_session),
163
- ):
164
- from core import record_comments
165
-
166
- scope = _scope_or_400(scope)
167
- _in_book(pid, session, scope)
168
- try:
169
- comment = record_comments.add_comment(
170
- session.runtime,
171
- pid,
172
- (body or {}).get("body"),
173
- session.uname,
174
- session.user.get("name") or session.uname,
175
- scope=scope,
176
- )
177
- except ValueError as exc:
178
- raise err(400, "bad_comment", str(exc))
179
- except record_comments.CommentsUnavailable:
180
- raise _unavailable()
181
- return {"comment": comment}
182
-
183
-
184
- @router.delete("/records/{pid}/comments/{comment_id}")
185
- @router.delete("/customers/{pid}/comments/{comment_id}")
186
- def remove_comment(
187
- pid: int,
188
- comment_id: str,
189
- scope: str = Query(default="customer"),
190
- session: Session = Depends(require_session),
191
- ):
192
- from core import record_comments
193
-
194
- scope = _scope_or_400(scope)
195
- _in_book(pid, session, scope)
196
- try:
197
- deleted = record_comments.delete_comment(
198
- session.runtime,
199
- pid,
200
- comment_id,
201
- session.uname,
202
- admin=session.admin,
203
- scope=scope,
204
- )
205
- except record_comments.CommentForbidden:
206
- raise err(403, "comment_forbidden", "only the author may delete this comment")
207
- except record_comments.CommentsUnavailable:
208
- raise _unavailable()
209
- if not deleted:
210
- raise err(404, "comment_not_found", "that comment no longer exists")
211
- return {"ok": True, "id": comment_id}
 
1
+ """Record detail routes: durable comments, scoped to the caller's book — ON EVERY DATABASE.
2
+
3
+ ⭐ WAVE 19 (owner item 12). This file used to be the CUSTOMER record's comment routes with a
4
+ customer-shaped wall bolted to the module import line: `_in_book` asked
5
+ `routes_customers.allowed_pids` whatever surface the browser was on. Opening a PRODUCT record and
6
+ typing a comment therefore asked the customer book about a CRC32 hash of a SKU code, and the
7
+ panel answered "that customer is not in your book" — the owner's report. The dangerous half is
8
+ the one nobody sees: a hash that collides with a real partner id passes the wall, and the comment
9
+ is filed against somebody's customer where the whole team can read it.
10
+
11
+ THE SHAPE NOW: `?scope=` names the database (the same vocabulary `/workspace?scope=` and the
12
+ events route's `scopeKey` already speak), and `_pool_or_refuse` resolves BOTH halves of the wall
13
+ per scope — the GRANT and the ROW SET — by asking that topic's own route, never by re-deriving
14
+ one here:
15
+
16
+ customer / cohort `routes_customers.allowed_pids` behind the `customer_data` grant
17
+ product `routes_products.scoped_pool` behind the `product_data` grant
18
+ ut_<slug> `routes_tables.scoped_pids`, whose `_defn_or_refuse` IS the wall
19
+ (404 unknown / 403 not yours — a user table has no module grant).
20
+ ⭐ W33-T03/D-183: `scoped_pIDs`, not `scoped_pOOL` — the pool builds every
21
+ ROW to derive a pid set this module discards, and RAISES 409 on a
22
+ read-through grid past one window, which is why the record drawer painted
23
+ an error page on `ut_odoo_gl_lines`.
24
+
25
+ ⚠ THE PATH KEEPS ITS `/customers/` SEGMENT. It is the shipped URL and `verify_api.py`'s E1a
26
+ section pins it; the scope now travels beside it explicitly. A nicer noun is not worth churning
27
+ another session's gate mid-wave — the WALL is the query parameter, not the word.
28
+
29
+ ⚠ NO DEFAULT BEYOND THE LEGACY ONE. An absent `scope` means `customer`, which is what every
30
+ shipped client sent and what keeps the old callers byte-identical; an UNRECOGNISED scope is a
31
+ 400, never a silent fallback to the customer book (`routes_grid._scope_or_400`'s rule, and for
32
+ the same reason: a typo served as `customer` answers a question nobody asked).
33
+ """
34
+ from fastapi import APIRouter, Body, Depends, Query
35
+
36
+ from deps import Session, err, require_session
37
+
38
+ router = APIRouter(prefix="/api/v1")
39
+
40
+ #: The customer topic's two names — one book, two surfaces (the Cohort page is the customer table
41
+ #: over hand-curated sets). Mirrors `modules.cohort.LEGACY_SCOPES` / `core.record_comments`.
42
+ _CUSTOMER_SCOPES = ("", "customer", "cohort")
43
+
44
+
45
+ def _scope_or_400(raw):
46
+ scope = str(raw or "customer").strip().lower()
47
+ if scope in _CUSTOMER_SCOPES or scope == "product" or scope.startswith("ut_"):
48
+ return "customer" if scope in _CUSTOMER_SCOPES else scope
49
+ raise err(400, "bad_scope",
50
+ "scope must be customer, cohort, product or a ut_ database — refusing to guess")
51
+
52
+
53
+ def _pool_or_refuse(session: Session, scope: str):
54
+ """The pids this session may attach comments to ON THIS DATABASE — grant wall included.
55
+
56
+ Returns **`(pids, unbounded)`** — a 2-tuple on EVERY branch. Raises the topic's own 403/404/503,
57
+ so a caller who may not open the surface never learns anything about the row they asked about.
58
+
59
+ ⛔ `unbounded` is TRUE only when the row set could not be ENUMERATED (a read-through grid past
60
+ one window), never when it is merely EMPTY. Those are opposite answers and `scoped_pids` returns
61
+ `frozenset()` for both — see the branch below.
62
+ ⚠ THE SHAPE IS A CONTRACT EVEN THOUGH THIS FUNCTION IS PRIVATE, and it has two consumers that
63
+ do not travel together: `_in_book` here, and a NEGATIVE CONTROL in `aios-web/api/verify_scopes.py`
64
+ that REPLACES this function with its own lambda. A gate's test double is a caller
65
+ ([[test-double-patched-by-a-name-list]]); when this signature moved, that double kept returning a
66
+ bare frozenset and the section died on `ValueError: too many values to unpack` — no tally, no
67
+ failing name. Change the shape here and that double changes with it.
68
+ """
69
+ if scope == "product":
70
+ from routes_products import MODULE as PRODUCT_MODULE, scoped_pool
71
+
72
+ session.require(PRODUCT_MODULE)
73
+ pids, _team, _rows, _fields = scoped_pool(session)
74
+ # ⚠ `(pids, unbounded)` on EVERY branch. This one returned a bare frozenset for ten minutes
75
+ # after the `ut_` branch grew its second element, and `_in_book`'s unpack would have raised
76
+ # a `TypeError` — a 500 on every product comment — while both other branches worked. A
77
+ # return shape is a contract even when the function is private.
78
+ return (pids, False)
79
+ if scope.startswith("ut_"):
80
+ # No module grant exists for a user table — `_defn_or_refuse` inside `scoped_pids` IS
81
+ # the wall (creator or admin, fail-closed), and it answers 404 before 403 exactly as the
82
+ # rows routes do.
83
+ #
84
+ # ⭐⭐ W33-T03 / D-183 — `scoped_pids`, NOT `scoped_pool`, AND THAT ONE WORD IS THE BUG.
85
+ # `scoped_pool` builds every ROW to derive a pid set this function then throws away, and on
86
+ # a read-through grid larger than one window it RAISES `409 window_required`. So opening the
87
+ # record drawer on `ut_odoo_gl_lines` (975,137 rows) painted an error page — for a panel
88
+ # that renders comments about ONE row it already has. `scoped_pids` answers the identical
89
+ # question (its docstring: *"the pid set is IDENTICAL, not merely equivalent"*) and takes
90
+ # W31-T20's `limits` OUT-PARAMETER instead of raising, which is the same shape `/workspace`
91
+ # used to become openable on those two grids.
92
+ from routes_tables import scoped_pids
93
+
94
+ limits = []
95
+ pids, _fields, _defn = scoped_pids(session, scope, limits=limits)
96
+ # ⛔ AN EMPTY PID SET AND AN UNRESOLVABLE ONE ARE OPPOSITE ANSWERS, and collapsing them is
97
+ # how a fail-closed default becomes a lie. `scoped_pids` returns `frozenset()` BOTH for a
98
+ # database with no rows and for a read-through grid too big to enumerate — it distinguishes
99
+ # them by APPENDING R6's sentence to `limits`. Without this branch the drawer would move
100
+ # from a 409 error page to a 403 "not in your book" on a row the user is looking at, which
101
+ # is the same defect wearing a politer message ([[empty-answer-vs-unfinished-answer]]).
102
+ # ⚠ ADMITTING HERE IS NOT A WIDENING, and the ruling is wave 27 / D-72: on a `ut_*` database
103
+ # THE TENANT IS THE UNIT — `scoped_pool` itself carries "no per-row owner filter; the
104
+ # table-level wall is the WHOLE wall". `_defn_or_refuse` has already run inside
105
+ # `scoped_pids` and answered 404/403. The pid set was only ever an existence check.
106
+ return (pids, bool(limits))
107
+ from routes_customers import MODULE as CUSTOMER_MODULE, allowed_pids
108
+
109
+ session.require(CUSTOMER_MODULE)
110
+ return (frozenset(allowed_pids(session)), False)
111
+
112
+
113
+ def _in_book(pid, session, scope):
114
+ pids, unbounded = _pool_or_refuse(session, scope)
115
+ if unbounded:
116
+ # The table wall passed and the row set is larger than this process will enumerate. Said
117
+ # out loud rather than silently admitting: R6's second sentence is that a limit which
118
+ # cannot be removed gets REPORTED, and this is the one place the report has no envelope to
119
+ # ride in.
120
+ print(f"[records] {scope}: pid membership unresolved (read-through beyond one window) — "
121
+ f"admitting on the table wall alone, per D-72")
122
+ return
123
+ if pid not in pids:
124
+ # 403, not 404: the record may exist, but this session may not inspect it.
125
+ raise err(403, "out_of_scope", "that record is not in your book")
126
+
127
+
128
+ def _unavailable():
129
+ return err(
130
+ 503,
131
+ "store_unavailable",
132
+ "record comments are temporarily unavailable — no change was saved",
133
+ )
134
+
135
+
136
+ # ⭐ WAVE 21 (D-17): the CANONICAL path is /records/{pid}/comments — comments hang off a RECORD
137
+ # in whatever topic `?scope=` names, and the customer-flavoured noun was wave-19 residue (the
138
+ # wall was always the query param). The old path stays as an ALIAS because the shipped client
139
+ # still calls it; verify_api pins the canonical path AND that the alias answers, so removing
140
+ # the alias later is a decision, never an accident.
141
+ @router.get("/records/{pid}/comments")
142
+ @router.get("/customers/{pid}/comments")
143
+ def comments(pid: int, scope: str = Query(default="customer"),
144
+ session: Session = Depends(require_session)):
145
+ from core import record_comments
146
+
147
+ scope = _scope_or_400(scope)
148
+ _in_book(pid, session, scope)
149
+ try:
150
+ rows = record_comments.list_comments(session.runtime, pid, scope=scope)
151
+ except record_comments.CommentsUnavailable:
152
+ raise _unavailable()
153
+ return {"comments": rows}
154
+
155
+
156
+ @router.post("/records/{pid}/comments", status_code=201)
157
+ @router.post("/customers/{pid}/comments", status_code=201)
158
+ def create_comment(
159
+ pid: int,
160
+ body: dict = Body(default=None),
161
+ scope: str = Query(default="customer"),
162
+ session: Session = Depends(require_session),
163
+ ):
164
+ from core import record_comments
165
+
166
+ scope = _scope_or_400(scope)
167
+ _in_book(pid, session, scope)
168
+ try:
169
+ comment = record_comments.add_comment(
170
+ session.runtime,
171
+ pid,
172
+ (body or {}).get("body"),
173
+ session.uname,
174
+ session.user.get("name") or session.uname,
175
+ scope=scope,
176
+ )
177
+ except ValueError as exc:
178
+ raise err(400, "bad_comment", str(exc))
179
+ except record_comments.CommentsUnavailable:
180
+ raise _unavailable()
181
+ return {"comment": comment}
182
+
183
+
184
+ @router.delete("/records/{pid}/comments/{comment_id}")
185
+ @router.delete("/customers/{pid}/comments/{comment_id}")
186
+ def remove_comment(
187
+ pid: int,
188
+ comment_id: str,
189
+ scope: str = Query(default="customer"),
190
+ session: Session = Depends(require_session),
191
+ ):
192
+ from core import record_comments
193
+
194
+ scope = _scope_or_400(scope)
195
+ _in_book(pid, session, scope)
196
+ try:
197
+ deleted = record_comments.delete_comment(
198
+ session.runtime,
199
+ pid,
200
+ comment_id,
201
+ session.uname,
202
+ admin=session.admin,
203
+ scope=scope,
204
+ )
205
+ except record_comments.CommentForbidden:
206
+ raise err(403, "comment_forbidden", "only the author may delete this comment")
207
+ except record_comments.CommentsUnavailable:
208
+ raise _unavailable()
209
+ if not deleted:
210
+ raise err(404, "comment_not_found", "that comment no longer exists")
211
+ return {"ok": True, "id": comment_id}
api/routes_script_views.py CHANGED
@@ -1,439 +1,439 @@
1
- """routes_script_views.py — CONTRACT C3: a database View that is a PYTHON SCRIPT (R3 / R5 / R10).
2
-
3
- Owner item 6, verbatim (2026-08-18): *"Add code script as an interface (database View) so a user
4
- can build whatever they want through the Agent chat interface. be able to create any dashboard
5
- they want. User should have the ability to see the code AND the dashboard output of course… Limit
6
- the code script View per database… Any agent can add into more AI script, so we can see different
7
- versions or different things the AI code for us."*
8
-
9
- GET /api/v1/script-views?database=K the views bound to ONE database
10
- POST /api/v1/script-views create one {database, name?, source}
11
- GET /api/v1/script-views/{id} one view, its source and its history
12
- PUT /api/v1/script-views/{id} a NEW VERSION of the source
13
- DELETE /api/v1/script-views/{id} drop it
14
- POST /api/v1/script-views/{id}/run run it -> {ok, spec | error, stdout, ms}
15
- POST /api/v1/script-views/{id}/revert go back to an earlier version {version}
16
-
17
- ⭐⭐ **R3 IS "SCOPED, NOT CAPPED", AND THE TWO HALVES POINT OPPOSITE WAYS.** *Scoped*: a view may
18
- read ONLY the database it lives in, and a script that names another database is REFUSED with a
19
- message naming both. *Not capped*: there is **no limit on how many script views a database may
20
- carry**, because that is how an agent offers three attempts and the owner picks one. So nothing
21
- below counts views. What IS bounded is what makes them big — one source is capped, and one view's
22
- edit history is capped and REPORTS what it dropped.
23
-
24
- ⛔ **THE RUN IS THE CALLER'S, NEVER THE AUTHOR'S (R5).** `script_sandbox.run_view` is handed
25
- `session.user`, so a script written by an administrator and opened by a scoped analyst reads the
26
- ANALYST's rows. The author decides what the code does; the reader decides what it can see.
27
- ⚠ And the reverse case is safe rather than lucky: a narrowly-scoped author cannot write a script
28
- that exfiltrates anything, because the only thing a script can return is a render spec drawn on
29
- the screen of the person who ran it. There is no network, no file and no second reader.
30
-
31
- ⛔ **`run` IS `def`, NOT `async def`.** It waits on a subprocess for up to ten seconds; as a
32
- coroutine that would block the event loop for every other request in the container. FastAPI runs a
33
- plain `def` in the threadpool, which is what makes one slow script one slow REQUEST.
34
- """
35
- import threading
36
- from datetime import datetime, timezone
37
-
38
- from fastapi import APIRouter, Body, Depends
39
-
40
- from deps import Session, err, require_session
41
-
42
- router = APIRouter(prefix="/api/v1")
43
-
44
- #: The tenant's script views: `{id: record}`. Per tenant, so it rides `runtime.store_key`.
45
- VIEWS_KEY = "script_views"
46
-
47
- MAX_NAME = 80
48
- MAX_SOURCE_BYTES = 128 * 1024
49
-
50
- #: Edit history per view. ⚠ NOT a cap on the NUMBER of views (R3 forbids that) — a cap on how far
51
- #: back ONE view's source is kept. Past this the oldest go and `trimmed` counts them, so a reader
52
- #: can see the history is partial instead of concluding the view was only ever saved twice.
53
- MAX_HISTORY = 40
54
-
55
- #: ⛔ HOW MANY SCRIPTS MAY BE RUNNING IN THIS CONTAINER AT ONCE, and it is a REPORTED refusal
56
- #: rather than a queue. Each run is a real subprocess with a ten-second wall clock; without this,
57
- #: holding down refresh forks until the box gives up, and the tenant's ONE FastAPI process is what
58
- #: gives up. A 429 that says so is honest; an unbounded fork is not.
59
- MAX_CONCURRENT_RUNS = 4
60
- _RUN_SLOTS = threading.BoundedSemaphore(MAX_CONCURRENT_RUNS)
61
-
62
-
63
- def _now():
64
- return datetime.now(timezone.utc).isoformat(timespec="seconds")
65
-
66
-
67
- def _all(rt):
68
- """`{id: record}` for one tenant. `{}` on any failure — an unreadable bucket must degrade to
69
- "this database has no script views", never to a 500 on the view rail."""
70
- try:
71
- found = rt.get(VIEWS_KEY) or {}
72
- except Exception: # noqa: BLE001
73
- return {}
74
- return found if isinstance(found, dict) else {}
75
-
76
-
77
- def _database_ok(session, database):
78
- """Does this database EXIST, and may this caller read it? Answered by C1, never by a list.
79
-
80
- ⛔ `perm_scope.may_read` ALONE IS NOT ENOUGH and the reason is easy to miss: it answers True
81
- for an ADMIN on any key at all, including one no database answers to. So a create validated
82
- with `may_read` would let an administrator bind a view to a typo and leave an orphan nothing
83
- can ever run. `scoped_fields` is the cheap half of C1 (a `ut_*` definition, no rows) and it
84
- RAISES `UnknownTable`, which is exactly the question being asked.
85
- """
86
- import core.perm_scope as perm_scope
87
- try:
88
- perm_scope.scoped_fields(session.user, database, st=session.runtime)
89
- except perm_scope.UnknownTable:
90
- raise err(404, "no_database", f"there is no database '{database}' in this workspace")
91
- except perm_scope.Denied:
92
- raise err(403, "forbidden", f"your account may not read '{database}'")
93
- except perm_scope.Unresolvable as exc:
94
- # The database is real and cannot be served under this call's constraints. Standing rule
95
- # 1's second sentence: report the cause and the recommendation, never a bare refusal.
96
- raise err(409, "unresolvable", str(exc)) from None
97
-
98
-
99
- def _clean_source(raw, *, allow_empty=False):
100
- """The stored source, or a 400. `allow_empty` is CREATE's alone and the asymmetry is the point.
101
-
102
- ⭐⭐ W37-T41 — WHY CREATE MAY BE EMPTY AND SAVE MAY NOT.
103
- Picking "Custom View" in the mode picker mints the view immediately, before a line of code
104
- exists, so that `mode === 'script'` always implies a real id and no reader has to carry a
105
- "the id might be missing" branch (the create semantics handed to lane C in mailbox E-7).
106
- A view that has been created and not yet written is therefore a REAL, legible state: the editor
107
- shows its starter placeholder and the Run control is right there.
108
- A PUT is a different act. It appends a VERSION to a history capped at 40, and blanking a
109
- working script by saving nothing over it is not an edit anybody means to make. So the guard
110
- stays exactly where it was on that door, and a person who wants the view gone deletes it.
111
-
112
- ⛔ THIS WAS FOUND BY THE GATE, NOT BY READING. `core.script_sandbox.check_source("")` returns
113
- None, so "an empty script is storable" looked true and was written into a mailbox answer another
114
- lane was about to build on. The refusal was HERE, one layer above, in a function the sandbox
115
- knows nothing about. Two validators for one question, disagreeing
116
- [[one-question-two-normalizers]] — and the one that would have bitten a person is the one no
117
- unit of this feature was asserting.
118
- """
119
- source = str(raw or "")
120
- if not source.strip() and not allow_empty:
121
- raise err(400, "no_source", "a script view needs some code")
122
- if len(source.encode("utf-8", "replace")) > MAX_SOURCE_BYTES:
123
- raise err(413, "source_too_long",
124
- f"a script view is at most {MAX_SOURCE_BYTES // 1024} KB of code")
125
- return source
126
-
127
-
128
- def _row(rec, *, source=False):
129
- """One view as the list door reports it. ⚠ NO SOURCE unless asked: the rail lists names."""
130
- out = {"id": rec.get("id") or "", "database": rec.get("database") or "",
131
- "name": rec.get("name") or "", "author": rec.get("author") or "",
132
- "version": int(rec.get("version") or 1),
133
- "created": rec.get("created") or "", "updated": rec.get("updated") or "",
134
- "versions": len(rec.get("history") or []) + 1,
135
- "trimmed": int(rec.get("trimmed") or 0),
136
- # ⭐ W37-T46: which version this one was RESTORED from, when it was. Present on the
137
- # row (not only the history) because the editor's header is where a person reads "what
138
- # am I looking at", and "v5, restored from v2" is the sentence that makes a roll-back
139
- # legible as an event rather than as a coincidence of matching code.
140
- "restoredFrom": rec.get("restoredFrom")}
141
- if source:
142
- out["source"] = rec.get("source") or ""
143
- out["history"] = [{"version": int(h.get("version") or 0), "author": h.get("author") or "",
144
- "created": h.get("created") or "",
145
- "bytes": len(str(h.get("source") or "").encode("utf-8", "replace"))}
146
- for h in reversed(rec.get("history") or []) if isinstance(h, dict)]
147
- return out
148
-
149
-
150
- def _limits():
151
- return {"maxSourceBytes": MAX_SOURCE_BYTES, "maxName": MAX_NAME,
152
- "maxHistory": MAX_HISTORY, "maxConcurrentRuns": MAX_CONCURRENT_RUNS,
153
- # ⭐ SAID OUT LOUD, because R3's "not capped" half is the one a reader assumes wrong.
154
- "maxViewsPerDatabase": None}
155
-
156
-
157
- def _put(session, view_id, mutate):
158
- """Read-modify-write ONE view, synchronously — the client re-reads the rail immediately."""
159
- def _set(cur):
160
- cur = dict(cur or {})
161
- nxt = mutate(cur.get(view_id) if isinstance(cur.get(view_id), dict) else None)
162
- if nxt is None:
163
- cur.pop(view_id, None)
164
- else:
165
- cur[view_id] = nxt
166
- return cur
167
-
168
- session.runtime.update(VIEWS_KEY, _set, flush="sync")
169
-
170
-
171
- def _mine_or_admin(session, rec):
172
- """Who may EDIT or DELETE a view: its author, or an administrator.
173
-
174
- ⚠ RUNNING IS A DIFFERENT QUESTION and deliberately wider — anybody who may read the database
175
- may run any view on it, under their OWN scope. That is the whole of "so we can see different
176
- versions or different things the AI code for us": a colleague's attempt is worth nothing if
177
- only its author can open it.
178
- """
179
- if session.admin or str(rec.get("author") or "") == session.uname:
180
- return
181
- raise err(403, "forbidden", "only the author or an administrator can change this script view")
182
-
183
-
184
- # ── the routes ────────────────────────────────────────────────────────────────────────────────
185
- @router.get("/script-views")
186
- def list_script_views(database: str = "", session: Session = Depends(require_session)):
187
- """Every script view bound to ONE database, newest first. `database` is required."""
188
- key = str(database or "").strip()
189
- if not key:
190
- raise err(400, "no_database", "name the database whose script views you want")
191
- _database_ok(session, key)
192
- rows = [_row(rec) for rec in _all(session.runtime).values()
193
- if isinstance(rec, dict) and str(rec.get("database") or "") == key]
194
- rows.sort(key=lambda r: (r["created"], r["id"]), reverse=True)
195
- return {"database": key, "views": rows, "limits": _limits()}
196
-
197
-
198
- @router.post("/script-views")
199
- def create_script_view(body: dict = Body(default=None),
200
- session: Session = Depends(require_session)):
201
- """Create one. ⛔ THE SOURCE IS CHECKED BEFORE IT IS STORED, not first at run time.
202
-
203
- A script view that cannot be run is a broken feature the reader discovers by pressing a button,
204
- and the agent that wrote it is long gone by then. `check_source` is pure and costs no process,
205
- so the refusal arrives while the author still has the code in front of them.
206
- """
207
- import secrets # noqa: PLC0415
208
- import core.script_sandbox as sandbox # noqa: PLC0415
209
-
210
- body = body if isinstance(body, dict) else {}
211
- database = str(body.get("database") or "").strip()
212
- if not database:
213
- raise err(400, "no_database", "a script view is bound to one database")
214
- _database_ok(session, database)
215
- # ⭐ `allow_empty` is CREATE's alone - see `_clean_source`. The picker mints a view the
216
- # moment a person chooses the mode, so the empty source is the normal first state, not a slip.
217
- source = _clean_source(body.get("source"), allow_empty=True)
218
- refusal = sandbox.check_source(source)
219
- if refusal is not None:
220
- raise err(400, refusal.code, refusal.message)
221
-
222
- view_id = "sv_" + secrets.token_urlsafe(9)
223
- rec = {"id": view_id, "database": database,
224
- "name": " ".join(str(body.get("name") or "Script view").split())[:MAX_NAME],
225
- "source": source, "author": session.uname, "version": 1,
226
- "created": _now(), "updated": _now(), "history": [], "trimmed": 0}
227
- _put(session, view_id, lambda _prior: rec)
228
- fresh = _all(session.runtime).get(view_id)
229
- if not isinstance(fresh, dict):
230
- # The store took the write and did not record it. A 200 here would tell the author their
231
- # script was saved when it was not.
232
- raise err(503, "store_unavailable", "the script view was NOT created")
233
- return {"view": _row(fresh, source=True), "limits": _limits()}
234
-
235
-
236
- @router.get("/script-views/{view_id}")
237
- def get_script_view(view_id: str, session: Session = Depends(require_session)):
238
- """One view WITH its source and its edit history. Owner item 6's *"see the code"* half."""
239
- rec = _all(session.runtime).get(str(view_id))
240
- if not isinstance(rec, dict):
241
- raise err(404, "no_view", "there is no script view with that id")
242
- _database_ok(session, str(rec.get("database") or ""))
243
- return {"view": _row(rec, source=True), "limits": _limits()}
244
-
245
-
246
- @router.put("/script-views/{view_id}")
247
- def update_script_view(view_id: str, body: dict = Body(default=None),
248
- session: Session = Depends(require_session)):
249
- """A NEW VERSION of one view's source. The prior source is KEPT, never replaced in place."""
250
- import core.script_sandbox as sandbox # noqa: PLC0415
251
-
252
- view_id = str(view_id)
253
- rec = _all(session.runtime).get(view_id)
254
- if not isinstance(rec, dict):
255
- raise err(404, "no_view", "there is no script view with that id")
256
- _mine_or_admin(session, rec)
257
- body = body if isinstance(body, dict) else {}
258
- source = _clean_source(body.get("source"))
259
- refusal = sandbox.check_source(source)
260
- if refusal is not None:
261
- raise err(400, refusal.code, refusal.message)
262
-
263
- def _mutate(prior):
264
- prior = dict(prior or rec)
265
- history = list(prior.get("history") or [])
266
- history.append({"version": int(prior.get("version") or 1),
267
- "source": prior.get("source") or "",
268
- "author": prior.get("author") or "", "created": prior.get("updated") or ""})
269
- dropped = max(0, len(history) - MAX_HISTORY)
270
- prior["history"] = history[dropped:] if dropped else history
271
- prior["trimmed"] = int(prior.get("trimmed") or 0) + dropped
272
- prior["source"] = source
273
- prior["version"] = int(prior.get("version") or 1) + 1
274
- prior["updated"] = _now()
275
- if body.get("name"):
276
- prior["name"] = " ".join(str(body["name"]).split())[:MAX_NAME]
277
- return prior
278
-
279
- _put(session, view_id, _mutate)
280
- fresh = _all(session.runtime).get(view_id)
281
- if not isinstance(fresh, dict):
282
- raise err(503, "store_unavailable", "the new version was NOT saved")
283
- return {"view": _row(fresh, source=True), "limits": _limits()}
284
-
285
-
286
- @router.post("/script-views/{view_id}/revert")
287
- def revert_script_view(view_id: str, body: dict = Body(default=None),
288
- session: Session = Depends(require_session)):
289
- """Put one view back to an earlier version. D-338 / W37-T46.
290
-
291
- ⭐⭐ A ROLL-BACK IS A NEW VERSION, NEVER A DELETE OF THE ONES AFTER IT, and that is the whole
292
- point of the ticket rather than an implementation detail. The history is what lets a person
293
- trust an agent with their code: if reverting destroyed the versions it stepped over, one
294
- mistaken revert would cost exactly what the history existed to protect, and there would be no
295
- way back from the way back. So this appends, and `restoredFrom` records where the text came
296
- from — the same posture the agent-harness roll-back already takes, so a reader who has seen one
297
- is not surprised by the other.
298
-
299
- ⛔ IT LIVES ON THE SERVER BECAUSE THE HISTORY DOES. The list door deliberately serves history
300
- entries WITHOUT their source (`_row`: the rail lists names), so a client cannot assemble an old
301
- version's text to re-PUT it. Handing the source out just so the client could send it straight
302
- back would widen a payload for a round trip that does not need to exist, and would make the
303
- revert non-atomic: two calls, and a failure between them leaves the person looking at code that
304
- is not what is stored.
305
- ⚠ `_clean_source` is NOT re-run. The text being restored was already validated when it was
306
- first saved, and a source that a later, stricter rule would now reject is exactly the source a
307
- person is most likely to want back. `check_source` still runs, because the SANDBOX's refusal is
308
- about what the code would DO and that must never be bypassed by a route.
309
- """
310
- import core.script_sandbox as sandbox # noqa: PLC0415
311
-
312
- view_id = str(view_id)
313
- rec = _all(session.runtime).get(view_id)
314
- if not isinstance(rec, dict):
315
- raise err(404, "no_view", "there is no script view with that id")
316
- _mine_or_admin(session, rec)
317
- body = body if isinstance(body, dict) else {}
318
- try:
319
- want = int(body.get("version"))
320
- except (TypeError, ValueError):
321
- raise err(400, "no_version", "say which version to go back to") from None
322
-
323
- history = [h for h in (rec.get("history") or []) if isinstance(h, dict)]
324
- match = next((h for h in history if int(h.get("version") or 0) == want), None)
325
- if match is None:
326
- # ⛔ TWO REASONS A VERSION IS MISSING AND THEY ARE NOT THE SAME FACT. One was never
327
- # written; the other was TRIMMED off the 40-deep history and the record even counts how
328
- # many. Saying "that version is gone" for the first would be a lie a person could act on.
329
- trimmed = int(rec.get("trimmed") or 0)
330
- if trimmed and want <= trimmed:
331
- raise err(410, "version_trimmed",
332
- f"version {want} is older than this view's history keeps "
333
- f"({trimmed} earlier versions have been dropped)")
334
- raise err(404, "no_version", f"this view has no version {want}")
335
-
336
- source = str(match.get("source") or "")
337
- refusal = sandbox.check_source(source)
338
- if refusal is not None:
339
- # An older version that today's sandbox refuses. The person is told which version and why,
340
- # rather than being handed a 400 about code they did not just type.
341
- raise err(400, refusal.code, f"version {want} cannot be restored: {refusal.message}")
342
-
343
- def _mutate(prior):
344
- prior = dict(prior or rec)
345
- history_now = list(prior.get("history") or [])
346
- history_now.append({"version": int(prior.get("version") or 1),
347
- "source": prior.get("source") or "",
348
- "author": prior.get("author") or "",
349
- "created": prior.get("updated") or ""})
350
- dropped = max(0, len(history_now) - MAX_HISTORY)
351
- prior["history"] = history_now[dropped:] if dropped else history_now
352
- prior["trimmed"] = int(prior.get("trimmed") or 0) + dropped
353
- prior["source"] = source
354
- prior["version"] = int(prior.get("version") or 1) + 1
355
- prior["updated"] = _now()
356
- # ⚠ ON THE RECORD, so the history reads as WHAT HAPPENED rather than as a version that
357
- # mysteriously matches an older one. Without it a reader sees v5 and v2 with identical
358
- # code and no way to tell a revert from a coincidence.
359
- prior["restoredFrom"] = want
360
- return prior
361
-
362
- _put(session, view_id, _mutate)
363
- fresh = _all(session.runtime).get(view_id)
364
- if not isinstance(fresh, dict):
365
- raise err(503, "store_unavailable", "the roll-back was NOT saved")
366
- return {"view": _row(fresh, source=True), "restoredFrom": want, "limits": _limits()}
367
-
368
-
369
- @router.delete("/script-views/{view_id}")
370
- def delete_script_view(view_id: str, session: Session = Depends(require_session)):
371
- view_id = str(view_id)
372
- rec = _all(session.runtime).get(view_id)
373
- if not isinstance(rec, dict):
374
- raise err(404, "no_view", "there is no script view with that id")
375
- _mine_or_admin(session, rec)
376
- _put(session, view_id, lambda _prior: None)
377
- return {"deleted": view_id}
378
-
379
-
380
- @router.post("/script-views/{view_id}/run")
381
- def run_script_view(view_id: str, body: dict = Body(default=None),
382
- session: Session = Depends(require_session)):
383
- """CONTRACT C3's run door: `{ok, spec | error, stdout, ms}`.
384
-
385
- ⛔ `spec` IS A DESCRIPTION THE CLIENT DRAWS. It is never HTML and never text the browser
386
- executes — the sandbox refuses a spec carrying an `html`, `script`, `src`, `href` or `on*` key
387
- before this function ever sees it, so a renderer cannot be talked into running something by a
388
- script that was itself perfectly well behaved.
389
-
390
- ⛔ AND IT IS PLAIN `def`, NOT `async def` — see this module's header. A ten-second subprocess
391
- wait on the event loop is a ten-second outage for the whole container.
392
- """
393
- import core.script_sandbox as sandbox # noqa: PLC0415
394
-
395
- rec = _all(session.runtime).get(str(view_id))
396
- if not isinstance(rec, dict):
397
- raise err(404, "no_view", "there is no script view with that id")
398
- database = str(rec.get("database") or "")
399
- # A DRAFT run: the editor sends unsaved code so the author can try it before committing to it.
400
- # It is checked exactly as a stored one is, because "unsaved" is not a permission.
401
- draft = (body or {}).get("source") if isinstance(body, dict) else None
402
- source = _clean_source(draft) if draft else str(rec.get("source") or "")
403
- # ⚠ NO `_database_ok` CALL HERE. `run_view` asks C1 the identical question a line later, and
404
- # asking twice builds a registry topic's pool twice. The three refusals are translated below
405
- # instead, which is the same wall reached through the same door.
406
-
407
- if not _RUN_SLOTS.acquire(blocking=False):
408
- raise err(429, "busy",
409
- f"{MAX_CONCURRENT_RUNS} script views are already running on this server. "
410
- f"Try again in a moment")
411
- try:
412
- out = sandbox.run_view(session.user, database, source, st=session.runtime)
413
- finally:
414
- _RUN_SLOTS.release()
415
-
416
- # ⛔ C1'S THREE REFUSALS ARE HTTP STATUSES, NOT `ok:false`. "You may not read this database"
417
- # answered 200 would be a permission decision the client has to go looking for, and every
418
- # other door in this app answers 403 for it. Everything BELOW this line is a well-formed
419
- # request whose ANSWER is that the script did not produce a view — that is a 200 with
420
- # `ok:false`, the shape `routes_web_agent.test` already uses for the same reason.
421
- if out.get("code") == "unknown_table":
422
- raise err(404, "no_database", out.get("message") or f"there is no database '{database}'")
423
- if out.get("code") == "denied":
424
- raise err(403, "forbidden", out.get("message") or "your account may not read that database")
425
- if out.get("code") == "unresolvable":
426
- refusal = err(409, "unresolvable", out.get("message") or "these rows cannot be served")
427
- refusal.detail["error"]["limit"] = out.get("limit") or {}
428
- raise refusal
429
-
430
- answer = {"ok": bool(out.get("ok")), "spec": out.get("spec"),
431
- "stdout": out.get("stdout") or "", "truncated": bool(out.get("truncated")),
432
- "ms": int(out.get("ms") or 0), "code": out.get("code") or "",
433
- # ⭐ `caps` RIDES ON THE ANSWER (standing rule 1). On a POSIX host all three limits
434
- # were applied; on a Windows host the memory and CPU ones were not, and a screen
435
- # that claims an enforcement which did not happen is the failure the rule is about.
436
- "caps": out.get("caps") or {}}
437
- if not answer["ok"]:
438
- answer["error"] = out.get("message") or "the script view did not produce a view"
439
- return answer
 
1
+ """routes_script_views.py — CONTRACT C3: a database View that is a PYTHON SCRIPT (R3 / R5 / R10).
2
+
3
+ Owner item 6, verbatim (2026-08-18): *"Add code script as an interface (database View) so a user
4
+ can build whatever they want through the Agent chat interface. be able to create any dashboard
5
+ they want. User should have the ability to see the code AND the dashboard output of course… Limit
6
+ the code script View per database… Any agent can add into more AI script, so we can see different
7
+ versions or different things the AI code for us."*
8
+
9
+ GET /api/v1/script-views?database=K the views bound to ONE database
10
+ POST /api/v1/script-views create one {database, name?, source}
11
+ GET /api/v1/script-views/{id} one view, its source and its history
12
+ PUT /api/v1/script-views/{id} a NEW VERSION of the source
13
+ DELETE /api/v1/script-views/{id} drop it
14
+ POST /api/v1/script-views/{id}/run run it -> {ok, spec | error, stdout, ms}
15
+ POST /api/v1/script-views/{id}/revert go back to an earlier version {version}
16
+
17
+ ⭐⭐ **R3 IS "SCOPED, NOT CAPPED", AND THE TWO HALVES POINT OPPOSITE WAYS.** *Scoped*: a view may
18
+ read ONLY the database it lives in, and a script that names another database is REFUSED with a
19
+ message naming both. *Not capped*: there is **no limit on how many script views a database may
20
+ carry**, because that is how an agent offers three attempts and the owner picks one. So nothing
21
+ below counts views. What IS bounded is what makes them big — one source is capped, and one view's
22
+ edit history is capped and REPORTS what it dropped.
23
+
24
+ ⛔ **THE RUN IS THE CALLER'S, NEVER THE AUTHOR'S (R5).** `script_sandbox.run_view` is handed
25
+ `session.user`, so a script written by an administrator and opened by a scoped analyst reads the
26
+ ANALYST's rows. The author decides what the code does; the reader decides what it can see.
27
+ ⚠ And the reverse case is safe rather than lucky: a narrowly-scoped author cannot write a script
28
+ that exfiltrates anything, because the only thing a script can return is a render spec drawn on
29
+ the screen of the person who ran it. There is no network, no file and no second reader.
30
+
31
+ ⛔ **`run` IS `def`, NOT `async def`.** It waits on a subprocess for up to ten seconds; as a
32
+ coroutine that would block the event loop for every other request in the container. FastAPI runs a
33
+ plain `def` in the threadpool, which is what makes one slow script one slow REQUEST.
34
+ """
35
+ import threading
36
+ from datetime import datetime, timezone
37
+
38
+ from fastapi import APIRouter, Body, Depends
39
+
40
+ from deps import Session, err, require_session
41
+
42
+ router = APIRouter(prefix="/api/v1")
43
+
44
+ #: The tenant's script views: `{id: record}`. Per tenant, so it rides `runtime.store_key`.
45
+ VIEWS_KEY = "script_views"
46
+
47
+ MAX_NAME = 80
48
+ MAX_SOURCE_BYTES = 128 * 1024
49
+
50
+ #: Edit history per view. ⚠ NOT a cap on the NUMBER of views (R3 forbids that) — a cap on how far
51
+ #: back ONE view's source is kept. Past this the oldest go and `trimmed` counts them, so a reader
52
+ #: can see the history is partial instead of concluding the view was only ever saved twice.
53
+ MAX_HISTORY = 40
54
+
55
+ #: ⛔ HOW MANY SCRIPTS MAY BE RUNNING IN THIS CONTAINER AT ONCE, and it is a REPORTED refusal
56
+ #: rather than a queue. Each run is a real subprocess with a ten-second wall clock; without this,
57
+ #: holding down refresh forks until the box gives up, and the tenant's ONE FastAPI process is what
58
+ #: gives up. A 429 that says so is honest; an unbounded fork is not.
59
+ MAX_CONCURRENT_RUNS = 4
60
+ _RUN_SLOTS = threading.BoundedSemaphore(MAX_CONCURRENT_RUNS)
61
+
62
+
63
+ def _now():
64
+ return datetime.now(timezone.utc).isoformat(timespec="seconds")
65
+
66
+
67
+ def _all(rt):
68
+ """`{id: record}` for one tenant. `{}` on any failure — an unreadable bucket must degrade to
69
+ "this database has no script views", never to a 500 on the view rail."""
70
+ try:
71
+ found = rt.get(VIEWS_KEY) or {}
72
+ except Exception: # noqa: BLE001
73
+ return {}
74
+ return found if isinstance(found, dict) else {}
75
+
76
+
77
+ def _database_ok(session, database):
78
+ """Does this database EXIST, and may this caller read it? Answered by C1, never by a list.
79
+
80
+ ⛔ `perm_scope.may_read` ALONE IS NOT ENOUGH and the reason is easy to miss: it answers True
81
+ for an ADMIN on any key at all, including one no database answers to. So a create validated
82
+ with `may_read` would let an administrator bind a view to a typo and leave an orphan nothing
83
+ can ever run. `scoped_fields` is the cheap half of C1 (a `ut_*` definition, no rows) and it
84
+ RAISES `UnknownTable`, which is exactly the question being asked.
85
+ """
86
+ import core.perm_scope as perm_scope
87
+ try:
88
+ perm_scope.scoped_fields(session.user, database, st=session.runtime)
89
+ except perm_scope.UnknownTable:
90
+ raise err(404, "no_database", f"there is no database '{database}' in this workspace")
91
+ except perm_scope.Denied:
92
+ raise err(403, "forbidden", f"your account may not read '{database}'")
93
+ except perm_scope.Unresolvable as exc:
94
+ # The database is real and cannot be served under this call's constraints. Standing rule
95
+ # 1's second sentence: report the cause and the recommendation, never a bare refusal.
96
+ raise err(409, "unresolvable", str(exc)) from None
97
+
98
+
99
+ def _clean_source(raw, *, allow_empty=False):
100
+ """The stored source, or a 400. `allow_empty` is CREATE's alone and the asymmetry is the point.
101
+
102
+ ⭐⭐ W37-T41 — WHY CREATE MAY BE EMPTY AND SAVE MAY NOT.
103
+ Picking "Custom View" in the mode picker mints the view immediately, before a line of code
104
+ exists, so that `mode === 'script'` always implies a real id and no reader has to carry a
105
+ "the id might be missing" branch (the create semantics handed to lane C in mailbox E-7).
106
+ A view that has been created and not yet written is therefore a REAL, legible state: the editor
107
+ shows its starter placeholder and the Run control is right there.
108
+ A PUT is a different act. It appends a VERSION to a history capped at 40, and blanking a
109
+ working script by saving nothing over it is not an edit anybody means to make. So the guard
110
+ stays exactly where it was on that door, and a person who wants the view gone deletes it.
111
+
112
+ ⛔ THIS WAS FOUND BY THE GATE, NOT BY READING. `core.script_sandbox.check_source("")` returns
113
+ None, so "an empty script is storable" looked true and was written into a mailbox answer another
114
+ lane was about to build on. The refusal was HERE, one layer above, in a function the sandbox
115
+ knows nothing about. Two validators for one question, disagreeing
116
+ [[one-question-two-normalizers]] — and the one that would have bitten a person is the one no
117
+ unit of this feature was asserting.
118
+ """
119
+ source = str(raw or "")
120
+ if not source.strip() and not allow_empty:
121
+ raise err(400, "no_source", "a script view needs some code")
122
+ if len(source.encode("utf-8", "replace")) > MAX_SOURCE_BYTES:
123
+ raise err(413, "source_too_long",
124
+ f"a script view is at most {MAX_SOURCE_BYTES // 1024} KB of code")
125
+ return source
126
+
127
+
128
+ def _row(rec, *, source=False):
129
+ """One view as the list door reports it. ⚠ NO SOURCE unless asked: the rail lists names."""
130
+ out = {"id": rec.get("id") or "", "database": rec.get("database") or "",
131
+ "name": rec.get("name") or "", "author": rec.get("author") or "",
132
+ "version": int(rec.get("version") or 1),
133
+ "created": rec.get("created") or "", "updated": rec.get("updated") or "",
134
+ "versions": len(rec.get("history") or []) + 1,
135
+ "trimmed": int(rec.get("trimmed") or 0),
136
+ # ⭐ W37-T46: which version this one was RESTORED from, when it was. Present on the
137
+ # row (not only the history) because the editor's header is where a person reads "what
138
+ # am I looking at", and "v5, restored from v2" is the sentence that makes a roll-back
139
+ # legible as an event rather than as a coincidence of matching code.
140
+ "restoredFrom": rec.get("restoredFrom")}
141
+ if source:
142
+ out["source"] = rec.get("source") or ""
143
+ out["history"] = [{"version": int(h.get("version") or 0), "author": h.get("author") or "",
144
+ "created": h.get("created") or "",
145
+ "bytes": len(str(h.get("source") or "").encode("utf-8", "replace"))}
146
+ for h in reversed(rec.get("history") or []) if isinstance(h, dict)]
147
+ return out
148
+
149
+
150
+ def _limits():
151
+ return {"maxSourceBytes": MAX_SOURCE_BYTES, "maxName": MAX_NAME,
152
+ "maxHistory": MAX_HISTORY, "maxConcurrentRuns": MAX_CONCURRENT_RUNS,
153
+ # ⭐ SAID OUT LOUD, because R3's "not capped" half is the one a reader assumes wrong.
154
+ "maxViewsPerDatabase": None}
155
+
156
+
157
+ def _put(session, view_id, mutate):
158
+ """Read-modify-write ONE view, synchronously — the client re-reads the rail immediately."""
159
+ def _set(cur):
160
+ cur = dict(cur or {})
161
+ nxt = mutate(cur.get(view_id) if isinstance(cur.get(view_id), dict) else None)
162
+ if nxt is None:
163
+ cur.pop(view_id, None)
164
+ else:
165
+ cur[view_id] = nxt
166
+ return cur
167
+
168
+ session.runtime.update(VIEWS_KEY, _set, flush="sync")
169
+
170
+
171
+ def _mine_or_admin(session, rec):
172
+ """Who may EDIT or DELETE a view: its author, or an administrator.
173
+
174
+ ⚠ RUNNING IS A DIFFERENT QUESTION and deliberately wider — anybody who may read the database
175
+ may run any view on it, under their OWN scope. That is the whole of "so we can see different
176
+ versions or different things the AI code for us": a colleague's attempt is worth nothing if
177
+ only its author can open it.
178
+ """
179
+ if session.admin or str(rec.get("author") or "") == session.uname:
180
+ return
181
+ raise err(403, "forbidden", "only the author or an administrator can change this script view")
182
+
183
+
184
+ # ── the routes ────────────────────────────────────────────────────────────────────────────────
185
+ @router.get("/script-views")
186
+ def list_script_views(database: str = "", session: Session = Depends(require_session)):
187
+ """Every script view bound to ONE database, newest first. `database` is required."""
188
+ key = str(database or "").strip()
189
+ if not key:
190
+ raise err(400, "no_database", "name the database whose script views you want")
191
+ _database_ok(session, key)
192
+ rows = [_row(rec) for rec in _all(session.runtime).values()
193
+ if isinstance(rec, dict) and str(rec.get("database") or "") == key]
194
+ rows.sort(key=lambda r: (r["created"], r["id"]), reverse=True)
195
+ return {"database": key, "views": rows, "limits": _limits()}
196
+
197
+
198
+ @router.post("/script-views")
199
+ def create_script_view(body: dict = Body(default=None),
200
+ session: Session = Depends(require_session)):
201
+ """Create one. ⛔ THE SOURCE IS CHECKED BEFORE IT IS STORED, not first at run time.
202
+
203
+ A script view that cannot be run is a broken feature the reader discovers by pressing a button,
204
+ and the agent that wrote it is long gone by then. `check_source` is pure and costs no process,
205
+ so the refusal arrives while the author still has the code in front of them.
206
+ """
207
+ import secrets # noqa: PLC0415
208
+ import core.script_sandbox as sandbox # noqa: PLC0415
209
+
210
+ body = body if isinstance(body, dict) else {}
211
+ database = str(body.get("database") or "").strip()
212
+ if not database:
213
+ raise err(400, "no_database", "a script view is bound to one database")
214
+ _database_ok(session, database)
215
+ # ⭐ `allow_empty` is CREATE's alone - see `_clean_source`. The picker mints a view the
216
+ # moment a person chooses the mode, so the empty source is the normal first state, not a slip.
217
+ source = _clean_source(body.get("source"), allow_empty=True)
218
+ refusal = sandbox.check_source(source)
219
+ if refusal is not None:
220
+ raise err(400, refusal.code, refusal.message)
221
+
222
+ view_id = "sv_" + secrets.token_urlsafe(9)
223
+ rec = {"id": view_id, "database": database,
224
+ "name": " ".join(str(body.get("name") or "Script view").split())[:MAX_NAME],
225
+ "source": source, "author": session.uname, "version": 1,
226
+ "created": _now(), "updated": _now(), "history": [], "trimmed": 0}
227
+ _put(session, view_id, lambda _prior: rec)
228
+ fresh = _all(session.runtime).get(view_id)
229
+ if not isinstance(fresh, dict):
230
+ # The store took the write and did not record it. A 200 here would tell the author their
231
+ # script was saved when it was not.
232
+ raise err(503, "store_unavailable", "the script view was NOT created")
233
+ return {"view": _row(fresh, source=True), "limits": _limits()}
234
+
235
+
236
+ @router.get("/script-views/{view_id}")
237
+ def get_script_view(view_id: str, session: Session = Depends(require_session)):
238
+ """One view WITH its source and its edit history. Owner item 6's *"see the code"* half."""
239
+ rec = _all(session.runtime).get(str(view_id))
240
+ if not isinstance(rec, dict):
241
+ raise err(404, "no_view", "there is no script view with that id")
242
+ _database_ok(session, str(rec.get("database") or ""))
243
+ return {"view": _row(rec, source=True), "limits": _limits()}
244
+
245
+
246
+ @router.put("/script-views/{view_id}")
247
+ def update_script_view(view_id: str, body: dict = Body(default=None),
248
+ session: Session = Depends(require_session)):
249
+ """A NEW VERSION of one view's source. The prior source is KEPT, never replaced in place."""
250
+ import core.script_sandbox as sandbox # noqa: PLC0415
251
+
252
+ view_id = str(view_id)
253
+ rec = _all(session.runtime).get(view_id)
254
+ if not isinstance(rec, dict):
255
+ raise err(404, "no_view", "there is no script view with that id")
256
+ _mine_or_admin(session, rec)
257
+ body = body if isinstance(body, dict) else {}
258
+ source = _clean_source(body.get("source"))
259
+ refusal = sandbox.check_source(source)
260
+ if refusal is not None:
261
+ raise err(400, refusal.code, refusal.message)
262
+
263
+ def _mutate(prior):
264
+ prior = dict(prior or rec)
265
+ history = list(prior.get("history") or [])
266
+ history.append({"version": int(prior.get("version") or 1),
267
+ "source": prior.get("source") or "",
268
+ "author": prior.get("author") or "", "created": prior.get("updated") or ""})
269
+ dropped = max(0, len(history) - MAX_HISTORY)
270
+ prior["history"] = history[dropped:] if dropped else history
271
+ prior["trimmed"] = int(prior.get("trimmed") or 0) + dropped
272
+ prior["source"] = source
273
+ prior["version"] = int(prior.get("version") or 1) + 1
274
+ prior["updated"] = _now()
275
+ if body.get("name"):
276
+ prior["name"] = " ".join(str(body["name"]).split())[:MAX_NAME]
277
+ return prior
278
+
279
+ _put(session, view_id, _mutate)
280
+ fresh = _all(session.runtime).get(view_id)
281
+ if not isinstance(fresh, dict):
282
+ raise err(503, "store_unavailable", "the new version was NOT saved")
283
+ return {"view": _row(fresh, source=True), "limits": _limits()}
284
+
285
+
286
+ @router.post("/script-views/{view_id}/revert")
287
+ def revert_script_view(view_id: str, body: dict = Body(default=None),
288
+ session: Session = Depends(require_session)):
289
+ """Put one view back to an earlier version. D-338 / W37-T46.
290
+
291
+ ⭐⭐ A ROLL-BACK IS A NEW VERSION, NEVER A DELETE OF THE ONES AFTER IT, and that is the whole
292
+ point of the ticket rather than an implementation detail. The history is what lets a person
293
+ trust an agent with their code: if reverting destroyed the versions it stepped over, one
294
+ mistaken revert would cost exactly what the history existed to protect, and there would be no
295
+ way back from the way back. So this appends, and `restoredFrom` records where the text came
296
+ from — the same posture the agent-harness roll-back already takes, so a reader who has seen one
297
+ is not surprised by the other.
298
+
299
+ ⛔ IT LIVES ON THE SERVER BECAUSE THE HISTORY DOES. The list door deliberately serves history
300
+ entries WITHOUT their source (`_row`: the rail lists names), so a client cannot assemble an old
301
+ version's text to re-PUT it. Handing the source out just so the client could send it straight
302
+ back would widen a payload for a round trip that does not need to exist, and would make the
303
+ revert non-atomic: two calls, and a failure between them leaves the person looking at code that
304
+ is not what is stored.
305
+ ⚠ `_clean_source` is NOT re-run. The text being restored was already validated when it was
306
+ first saved, and a source that a later, stricter rule would now reject is exactly the source a
307
+ person is most likely to want back. `check_source` still runs, because the SANDBOX's refusal is
308
+ about what the code would DO and that must never be bypassed by a route.
309
+ """
310
+ import core.script_sandbox as sandbox # noqa: PLC0415
311
+
312
+ view_id = str(view_id)
313
+ rec = _all(session.runtime).get(view_id)
314
+ if not isinstance(rec, dict):
315
+ raise err(404, "no_view", "there is no script view with that id")
316
+ _mine_or_admin(session, rec)
317
+ body = body if isinstance(body, dict) else {}
318
+ try:
319
+ want = int(body.get("version"))
320
+ except (TypeError, ValueError):
321
+ raise err(400, "no_version", "say which version to go back to") from None
322
+
323
+ history = [h for h in (rec.get("history") or []) if isinstance(h, dict)]
324
+ match = next((h for h in history if int(h.get("version") or 0) == want), None)
325
+ if match is None:
326
+ # ⛔ TWO REASONS A VERSION IS MISSING AND THEY ARE NOT THE SAME FACT. One was never
327
+ # written; the other was TRIMMED off the 40-deep history and the record even counts how
328
+ # many. Saying "that version is gone" for the first would be a lie a person could act on.
329
+ trimmed = int(rec.get("trimmed") or 0)
330
+ if trimmed and want <= trimmed:
331
+ raise err(410, "version_trimmed",
332
+ f"version {want} is older than this view's history keeps "
333
+ f"({trimmed} earlier versions have been dropped)")
334
+ raise err(404, "no_version", f"this view has no version {want}")
335
+
336
+ source = str(match.get("source") or "")
337
+ refusal = sandbox.check_source(source)
338
+ if refusal is not None:
339
+ # An older version that today's sandbox refuses. The person is told which version and why,
340
+ # rather than being handed a 400 about code they did not just type.
341
+ raise err(400, refusal.code, f"version {want} cannot be restored: {refusal.message}")
342
+
343
+ def _mutate(prior):
344
+ prior = dict(prior or rec)
345
+ history_now = list(prior.get("history") or [])
346
+ history_now.append({"version": int(prior.get("version") or 1),
347
+ "source": prior.get("source") or "",
348
+ "author": prior.get("author") or "",
349
+ "created": prior.get("updated") or ""})
350
+ dropped = max(0, len(history_now) - MAX_HISTORY)
351
+ prior["history"] = history_now[dropped:] if dropped else history_now
352
+ prior["trimmed"] = int(prior.get("trimmed") or 0) + dropped
353
+ prior["source"] = source
354
+ prior["version"] = int(prior.get("version") or 1) + 1
355
+ prior["updated"] = _now()
356
+ # ⚠ ON THE RECORD, so the history reads as WHAT HAPPENED rather than as a version that
357
+ # mysteriously matches an older one. Without it a reader sees v5 and v2 with identical
358
+ # code and no way to tell a revert from a coincidence.
359
+ prior["restoredFrom"] = want
360
+ return prior
361
+
362
+ _put(session, view_id, _mutate)
363
+ fresh = _all(session.runtime).get(view_id)
364
+ if not isinstance(fresh, dict):
365
+ raise err(503, "store_unavailable", "the roll-back was NOT saved")
366
+ return {"view": _row(fresh, source=True), "restoredFrom": want, "limits": _limits()}
367
+
368
+
369
+ @router.delete("/script-views/{view_id}")
370
+ def delete_script_view(view_id: str, session: Session = Depends(require_session)):
371
+ view_id = str(view_id)
372
+ rec = _all(session.runtime).get(view_id)
373
+ if not isinstance(rec, dict):
374
+ raise err(404, "no_view", "there is no script view with that id")
375
+ _mine_or_admin(session, rec)
376
+ _put(session, view_id, lambda _prior: None)
377
+ return {"deleted": view_id}
378
+
379
+
380
+ @router.post("/script-views/{view_id}/run")
381
+ def run_script_view(view_id: str, body: dict = Body(default=None),
382
+ session: Session = Depends(require_session)):
383
+ """CONTRACT C3's run door: `{ok, spec | error, stdout, ms}`.
384
+
385
+ ⛔ `spec` IS A DESCRIPTION THE CLIENT DRAWS. It is never HTML and never text the browser
386
+ executes — the sandbox refuses a spec carrying an `html`, `script`, `src`, `href` or `on*` key
387
+ before this function ever sees it, so a renderer cannot be talked into running something by a
388
+ script that was itself perfectly well behaved.
389
+
390
+ ⛔ AND IT IS PLAIN `def`, NOT `async def` — see this module's header. A ten-second subprocess
391
+ wait on the event loop is a ten-second outage for the whole container.
392
+ """
393
+ import core.script_sandbox as sandbox # noqa: PLC0415
394
+
395
+ rec = _all(session.runtime).get(str(view_id))
396
+ if not isinstance(rec, dict):
397
+ raise err(404, "no_view", "there is no script view with that id")
398
+ database = str(rec.get("database") or "")
399
+ # A DRAFT run: the editor sends unsaved code so the author can try it before committing to it.
400
+ # It is checked exactly as a stored one is, because "unsaved" is not a permission.
401
+ draft = (body or {}).get("source") if isinstance(body, dict) else None
402
+ source = _clean_source(draft) if draft else str(rec.get("source") or "")
403
+ # ⚠ NO `_database_ok` CALL HERE. `run_view` asks C1 the identical question a line later, and
404
+ # asking twice builds a registry topic's pool twice. The three refusals are translated below
405
+ # instead, which is the same wall reached through the same door.
406
+
407
+ if not _RUN_SLOTS.acquire(blocking=False):
408
+ raise err(429, "busy",
409
+ f"{MAX_CONCURRENT_RUNS} script views are already running on this server. "
410
+ f"Try again in a moment")
411
+ try:
412
+ out = sandbox.run_view(session.user, database, source, st=session.runtime)
413
+ finally:
414
+ _RUN_SLOTS.release()
415
+
416
+ # ⛔ C1'S THREE REFUSALS ARE HTTP STATUSES, NOT `ok:false`. "You may not read this database"
417
+ # answered 200 would be a permission decision the client has to go looking for, and every
418
+ # other door in this app answers 403 for it. Everything BELOW this line is a well-formed
419
+ # request whose ANSWER is that the script did not produce a view — that is a 200 with
420
+ # `ok:false`, the shape `routes_web_agent.test` already uses for the same reason.
421
+ if out.get("code") == "unknown_table":
422
+ raise err(404, "no_database", out.get("message") or f"there is no database '{database}'")
423
+ if out.get("code") == "denied":
424
+ raise err(403, "forbidden", out.get("message") or "your account may not read that database")
425
+ if out.get("code") == "unresolvable":
426
+ refusal = err(409, "unresolvable", out.get("message") or "these rows cannot be served")
427
+ refusal.detail["error"]["limit"] = out.get("limit") or {}
428
+ raise refusal
429
+
430
+ answer = {"ok": bool(out.get("ok")), "spec": out.get("spec"),
431
+ "stdout": out.get("stdout") or "", "truncated": bool(out.get("truncated")),
432
+ "ms": int(out.get("ms") or 0), "code": out.get("code") or "",
433
+ # ⭐ `caps` RIDES ON THE ANSWER (standing rule 1). On a POSIX host all three limits
434
+ # were applied; on a Windows host the memory and CPU ones were not, and a screen
435
+ # that claims an enforcement which did not happen is the failure the rule is about.
436
+ "caps": out.get("caps") or {}}
437
+ if not answer["ok"]:
438
+ answer["error"] = out.get("message") or "the script view did not produce a view"
439
+ return answer
api/routes_shares.py CHANGED
@@ -1,452 +1,458 @@
1
- """routes_shares.py — the manage-access surface (wave 20, owner ruling R10, contract C-SHARE).
2
-
3
- GET /api/v1/share/{kind}/{oid} -> {owner, entries:[{user,role}], mayAdminister, people}
4
- PUT /api/v1/share/{kind}/{oid} <- {entries:[{user,role}]} (REPLACES the set)
5
- GET /api/v1/share/mine -> {view:[id], folder:[id], database:[id]}
6
-
7
- `kind` ∈ view | folder | database | field. Roles are `view` | `edit` — the same two words the
8
- view rail already speaks, now extended to folders, databases and COLUMNS so there is ONE
9
- vocabulary in the UI (R10: "the same picker views use").
10
-
11
- ⭐⭐ **W38-T16 — `field` IS THE FOURTH KIND, AND ITS `oid` IS TOPIC-QUALIFIED: `"<table_key>:<field_key>"`**
12
- (`shares.field_oid`). A bare column key repeats across databases — `notes` exists on a dozen —
13
- so a grant stored under one would admit the grantee to every `notes` column in the tenant at
14
- once. ⛔ THREE functions in this file branch on kind and ALL THREE need the new one, which is not
15
- obvious because only two of them fail loudly: `_owns_object` (without it a column's own creator
16
- is 404'd trying to share the thing they just made) and `_object_ref` (without it `route` is None,
17
- `_notify_new_grantees` returns early, and the grantee is **granted and never told** — owner item
18
- 18's silent half, reopened one kind over). `_can_see_object` stays deliberately CLOSED for
19
- anything that is not a view.
20
-
21
- ⛔ **RE-SHARING IS THE OWNER'S, AND THAT IS ENFORCED HERE, NOT IN THE CLIENT.** `PUT` requires
22
- `shares.may_administer` (owner or admin). A collaborator with `edit` may change an object's
23
- CONTENT and may not change who else can reach it — otherwise anyone you shared a view with could
24
- widen it to everyone, or grant themselves ownership and lock you out. The client greys the editor
25
- for non-administrators; that is a courtesy, and this check is the wall.
26
-
27
- ⚠ **THE GRANT NEVER WIDENS PAST THE MODULE WALL — ON A GOVERNED MODULE.** `*` ("everyone") means
28
- every account that can already open the surface: `require_session` plus the topic's own gate run
29
- first, and for `customer_data` / `product_data` the receiver's own row scope and hidden-field
30
- closure run BEFORE any foreign view is merged. Sharing there can only narrow-or-equal the set that
31
- could already reach the data ([[aios-permissioning]]).
32
-
33
- ⛔⛔ **AND THAT SENTENCE IS FALSE FOR `kind='database'`, WHICH IS WHY IT NOW SAYS "ON A GOVERNED
34
- MODULE" (W32-T26, audit S-8).** `routes_admin._PERM_MODULES` is `("customer_data","product_data")`
35
- and `_clean_perms` **400s** on anything else, so **no row filter and no hidden field can even be
36
- DECLARED for a `ut_*` database** — `routes_tables.py` makes zero `perm_scope` calls and passes
37
- `hidden_keys=frozenset()`. There is no module wall behind a user table for a grant to be bounded
38
- by: **this registry IS the wall.** So a `database` grant is ALL-OR-NOTHING — every row, every
39
- column — and an `*` database grant admits every account in the tenant to all of it.
40
- That is a real capability, deliberately kept; what was wrong was a docstring promising a second
41
- wall that does not exist for this kind. Scoping user tables is booked, not done
42
- (`waves/wave32/sharing-audit.md` S-8).
43
-
44
- ⚠ **TWO SYSTEMS ANSWER "IS THIS SHARED", AND THEY ARE NOT THE SAME ONE (audit S-4).** THIS
45
- registry decides who appears in *"Shared with me"* and who may re-share. **`table_store.is_shared`
46
- — the view's own `permissions` — is what actually decides who may OPEN a view.** A grant here
47
- whose object is invisible under that one is a row in a list that opens a refusal, which is what
48
- made item 18 worth auditing. `_entries_or_400` closes the common cause (a name nobody has), but
49
- the two vocabularies are still two.
50
- """
51
- from fastapi import APIRouter, Body, Depends
52
-
53
- import core.shares as shares
54
- import core.users as users
55
- from deps import Session, err, require_session
56
- # ⭐ W32-T28 (C3) — the SHARE notification's topic word, imported from the module that CLASSIFIES
57
- # it (`routes_alerts.notification_view`) rather than typed again here. The producer and the
58
- # reader agreeing about one string is the whole difference between an Inbox row that opens the
59
- # shared database and one that is quietly unclickable.
60
- from routes_alerts import SHARE_TOPIC as _SHARE_TOPIC
61
-
62
- router = APIRouter(prefix="/api/v1")
63
-
64
-
65
- def _kind_or_400(raw):
66
- try:
67
- return shares._check_kind(raw)
68
- except ValueError as e:
69
- raise err(400, "bad_kind", str(e))
70
-
71
-
72
- # ── ⭐⭐ WAVE 32 · T26 (owner item 18, ruling R12) — THE WALL THIS FILE SAID IT HAD ─────────────
73
- #
74
- # `put_share`'s comment used to justify the first-claim rule with *"reaching this route at all
75
- # means passing the surface's own wall"*. **There was no such wall.** `kind` and `oid` are free
76
- # strings off the URL and the only dependency was `require_session`, so any signed-in account
77
- # could `PUT` a grant on an id it had never seen. Because the 403 sat behind `if rec["owner"]`,
78
- # an object with no grant record skipped the check entirely and the caller was stamped OWNER —
79
- # sticky, so **the real creator was then refused on their own view, permanently.** Driven, not
80
- # argued: `waves/wave32/sharing-audit.md` S-1 carries the four-step transcript.
81
- #
82
- # ⚠ AND IT WAS SILENT ON BOTH SIDES. The claimant does not even see the object in their own
83
- # "Shared with me" (`shared_with` excludes what you own), so nothing appears anywhere until the
84
- # victim next opens the dialog.
85
-
86
- #: The built-in grid topics. A view or folder lives in `{topic}_table_workspace`, and the share
87
- #: route is not told which topic — so resolving one means asking each.
88
- _BUILTIN_TOPICS = ("customer", "product")
89
-
90
-
91
- def _topics(session):
92
- """Every topic whose workspace could hold a view or folder for this tenant.
93
-
94
- ⚠ `all_defs`, never `all_tables` — the latter is the whole 28.6 MB row payload (~703 ms on
95
- tenant #0) to answer a question about KEYS (D-185).
96
- """
97
- try:
98
- import core.user_tables as ut
99
- return (*_BUILTIN_TOPICS, *(ut.all_defs(st=session.runtime) or {}))
100
- except Exception: # noqa: BLE001
101
- return _BUILTIN_TOPICS
102
-
103
-
104
- def _owns_object(session, kind, oid):
105
- """May this caller CLAIM an object that has no grant record yet — i.e. do they own it?
106
-
107
- ⛔ THIS GUARDS THE CLAIM, NOT THE READ, AND THAT IS DELIBERATE. Resolving a view means asking
108
- each topic's workspace in turn, which is N store reads; making every share call pay that
109
- would put a loop on a route the manage-access dialog opens. The dangerous path is the one
110
- where a caller is about to be stamped OWNER of something nobody owns — so the resolution runs
111
- exactly there, and the common path (a record exists, `may_administer` decides) is untouched.
112
- """
113
- if session.admin:
114
- return True
115
- if kind == "field":
116
- # ⭐⭐ W38-T16 — A COLUMN'S OWNER IS ITS `createdBy`, WHICH THE CREATE DOOR ALREADY STAMPS
117
- # (`routes_tables.patch_shared_cell`) and the DELETE door already reads as its wall (R8 /
118
- # D-172: creator-or-admin). Read from the same place by all three, so a column cannot be
119
- # deletable by one person and shareable by another.
120
- # ⚠ THIS BRANCH IS NOT OPTIONAL AND ITS ABSENCE FAILS SILENTLY IN THE WORST DIRECTION:
121
- # a brand-new column has no grant record, so `put_share` falls to this predicate — and
122
- # without it the column's own creator is answered `404 no_object` on the first attempt to
123
- # share the thing they just made.
124
- table_key, field_key = shares.split_field_oid(oid)
125
- if not table_key:
126
- return False
127
- try:
128
- import core.shared_overlay as shared_overlay
129
- defn = (shared_overlay.fields(table_key, st=session.runtime) or {}).get(field_key)
130
- except Exception: # noqa: BLE001
131
- return False
132
- return bool(defn) and str(defn.get("createdBy") or "") == str(session.uname)
133
- if kind == "database":
134
- # ⚠ `may_open` is THE resolver for a user table (its own docstring says so) and already
135
- # admits creator, admin, or a `database` grantee. Re-implementing "who owns a table"
136
- # here would be the second definition this wave keeps finding.
137
- try:
138
- import core.user_tables as ut
139
- return bool(ut.may_open(oid, session.uname, is_admin=session.admin,
140
- st=session.runtime))
141
- except Exception: # noqa: BLE001
142
- return False
143
- try:
144
- import core.table_store as table_store
145
- except Exception: # noqa: BLE001
146
- return False
147
- for topic in _topics(session):
148
- try:
149
- ops = table_store.make(f"{topic}_table_workspace", st=session.runtime)
150
- hit = ops.find_view(oid) if kind == "view" else ops.find_folder(oid)
151
- except Exception: # noqa: BLE001
152
- continue
153
- if hit:
154
- # `find_view`/`find_folder` answer `(owner_username, …)`. The claim belongs to the
155
- # person whose personal stratum holds it — anybody else reaching this line is
156
- # exactly the case S-1 describes.
157
- return str(hit[0]) == str(session.uname)
158
- return False
159
-
160
-
161
- def _can_see_object(session, kind, oid):
162
- """May this caller READ an object's grant list — i.e. can they reach the object at all?
163
-
164
- ⛔⛔ THIS IS DELIBERATELY WIDER THAN {@link _owns_object}, AND CONFLATING THE TWO IS A
165
- REGRESSION I SHIPPED AND CAUGHT. The first version of T26 guarded BOTH doors with the
166
- ownership test, which reads sensibly and is wrong for the read, because **`find_view` searches
167
- PERSONAL STRATA ONLY** (its own docstring says so). So a view living in alice's stratum with
168
- `permissions.edit = "collaborative"` and no grant record yet — a view bob **can open and edit
169
- in the grid** — answered `404` when bob opened its manage-access dialog. Measured before
170
- fixing: `table_store._may_see(view, "bob") is True` while `GET /share/view/vc` said
171
- `404 no_object`.
172
- ⚠ THAT IS THE AUDIT'S OWN S-4 BITING THE AUDIT'S OWN FIX: two systems answer "is this shared",
173
- and the wall consulted the grant registry (system A) plus stratum ownership, never the view's
174
- `permissions` (system B) — which is the one that actually decides who may OPEN it.
175
- ⚠ And it hides the ANSWER, not just the editor. `ViewSidebar`'s Share row is deliberately not
176
- gated on edit rights because *"hiding the row from everyone else would hide the ANSWER too —
177
- 'who has this?' is a fair question for anyone the view was shared with"*. A 404 there tells a
178
- legitimate collaborator their view does not exist.
179
-
180
- ⛔ THE CLAIM KEEPS THE NARROW TEST. Being able to SEE an object must not let you become its
181
- owner — that is S-1, and widening this predicate onto `put_share` would re-open it.
182
- """
183
- if _owns_object(session, kind, oid):
184
- return True
185
- if kind != "view":
186
- # A folder carries no per-object visibility flag of its own, and a database's `may_open`
187
- # (inside `_owns_object`) already admits grantees. Nothing wider to ask.
188
- # ⭐ W38-T16 — AND `field` KEEPS THIS CLOSED, DELIBERATELY. A grantee never reaches here:
189
- # `get_share` tests `role is None` first and a grant answers a role, so the only caller
190
- # left is an account with no relationship to the column at all. Widening it would let any
191
- # signed-in session enumerate who holds which column on a database they cannot open.
192
- return False
193
- try:
194
- import core.table_store as table_store
195
- for topic in _topics(session):
196
- hit = table_store.make(f"{topic}_table_workspace", st=session.runtime).find_view(oid)
197
- if hit:
198
- return bool(table_store._may_see(hit[1] if len(hit) > 1 else {},
199
- session.uname, is_admin=session.admin))
200
- except Exception: # noqa: BLE001
201
- return False
202
- return False
203
-
204
-
205
- def _entries_or_400(session, entries):
206
- """Validate a grant list against the tenant's REAL, ACTIVE accounts — and refuse BY NAME.
207
-
208
- ⛔ `core.shares._clean_entries` silently drops junk, and its docstring argues that correctly:
209
- a UI mid-save must not lose the whole list to one malformed row. **But it validates the SHAPE
210
- of a string and the role word — never that the user EXISTS, is ACTIVE, or is in this tenant**,
211
- so a typo'd name is stored, reported as a successful save, and never reaches anybody. The
212
- sharer believes the person has access. That is item 18's plain reading.
213
- ⚠ The correct population is computed THREE FUNCTIONS BELOW and served to the picker
214
- (`_people`). One route, two populations, and the write door was the permissive one.
215
- ⚠ `*` (everyone) is not a user and is admitted deliberately — it is R10's vocabulary for
216
- "every account that can already open the surface".
217
- """
218
- known = {p["username"].strip().lower() for p in _people(session.tenant)}
219
- unknown = []
220
- for e in entries or ():
221
- if not isinstance(e, dict):
222
- continue
223
- user = str(e.get("user") or "").strip().lower()
224
- if user and user != shares.EVERYONE and user not in known:
225
- unknown.append(user)
226
- if unknown:
227
- raise err(400, "unknown_people",
228
- "no active account in this workspace is named "
229
- + ", ".join(sorted(set(unknown)))
230
- + ". Nothing was shared. Pick people from the list rather than typing a name.")
231
-
232
-
233
- @router.get("/share/mine")
234
- def my_shares(session: Session = Depends(require_session)):
235
- """Everything shared WITH me, by kind — the "Shared with me" rail section (R10).
236
-
237
- Registered before `/share/{kind}/{oid}` so the literal path wins the match; FastAPI resolves
238
- in declaration order and `mine` would otherwise be read as a `kind`, answering 400 for a URL
239
- that is not malformed at all.
240
- """
241
- return shares.shared_with(session.uname, st=session.runtime)
242
-
243
-
244
- @router.get("/share/{kind}/{oid}")
245
- def get_share(kind: str, oid: str, session: Session = Depends(require_session)):
246
- kind = _kind_or_400(kind)
247
- rec = shares.grants(kind, oid, st=session.runtime)
248
- role = shares.role_for(kind, oid, session.uname, is_admin=session.admin, st=session.runtime)
249
- may_admin = shares.may_administer(kind, oid, session.uname, is_admin=session.admin,
250
- st=session.runtime)
251
- # ⭐ W32-T26 (audit S-3) A STRANGER LEARNS NOTHING. This route used to answer for ANY id:
252
- # who owns it, everyone it is granted to, and the tenant's whole username↔name directory —
253
- # to any signed-in session, about objects it cannot open. Now a caller with no role on an
254
- # object must prove they can reach it, and gets a 404 otherwise: the same answer a
255
- # non-existent id gives, so the route cannot be used to probe which ids are real.
256
- # `role is None` is the cheap pre-test, so the N-topic resolution below runs only for a
257
- # caller who has no relationship with the object at all.
258
- if role is None and not _can_see_object(session, kind, oid):
259
- raise err(404, "no_object", "no such item, or it is not shared with this account")
260
- return {
261
- **rec,
262
- "role": role,
263
- "mayAdminister": may_admin,
264
- # WAVE 21 (C1 identity fix): grant entries BIND on USERNAMES, so the picker must carry
265
- # them. `assignable_people` serves bare display names because `user`-kind CELLS store
266
- # display names — that list's shape cannot change without migrating cell values — so
267
- # this route serves objects of its own. Existing grants that were written as lowercased
268
- # display names are normalised by the wave-21 cleanup script.
269
- # ⭐ W32-T26 (audit S-3) — the roster is the EDITOR's data, so it rides only for a caller
270
- # who may open the editor. A read-only grantee gets the grant list (their fair question is
271
- # "who else has this?") and not a directory of every account in the workspace.
272
- "people": _people(session.tenant) if may_admin else [],
273
- }
274
-
275
-
276
- def _people(tenant):
277
- """[{username, name}] for this tenant same population as `assignable_people`, with the
278
- BINDING identity alongside the display one."""
279
- try:
280
- reg = users.registry() or {}
281
- except Exception:
282
- return []
283
- want = str(tenant or '').strip().lower()
284
- out = []
285
- for uname, u in reg.items():
286
- if not isinstance(u, dict) or u.get('active') is False:
287
- continue
288
- if want and str(u.get('tenant') or 'royal-imports').strip().lower() != want:
289
- continue
290
- out.append({"username": str(uname), "name": str(u.get('name') or uname)})
291
- return sorted(out, key=lambda p: p["name"].lower())
292
-
293
-
294
- @router.put("/share/{kind}/{oid}")
295
- def put_share(kind: str, oid: str, body: dict = Body(default=None),
296
- session: Session = Depends(require_session)):
297
- kind = _kind_or_400(kind)
298
- body = body or {}
299
- rec = shares.grants(kind, oid, st=session.runtime)
300
- # An object with NO grant record yet has no owner — the first person to share it claims it.
301
- # That is safe because reaching this route at all means passing the surface's own wall, and
302
- # the alternative (refusing until somebody seeds an owner) would make a brand-new folder
303
- # unshareable by the person who just made it.
304
- if rec["owner"]:
305
- if not shares.may_administer(kind, oid, session.uname, is_admin=session.admin,
306
- st=session.runtime):
307
- raise err(403, "not_owner",
308
- "only the owner of this item (or an administrator) can change who it is "
309
- "shared with")
310
- # ⛔⛔ W32-T26 (audit S-1) — THE CLAIM NOW HAS A PRECONDITION. An object with no grant record
311
- # is still claimed by the first person to share it — that rule is right, and refusing until
312
- # somebody seeds an owner would make a brand-new folder unshareable by the person who just
313
- # made it. What was missing is the half the old comment ASSERTED and the code never did: the
314
- # claimant has to be able to reach the object. Without this, any signed-in account could
315
- # stamp itself owner of an id it had never seen and lock the real creator out for good.
316
- elif not _owns_object(session, kind, oid):
317
- raise err(404, "no_object", "no such item, or it is not shared with this account")
318
- entries = body.get("entries")
319
- if not isinstance(entries, list):
320
- raise err(400, "bad_entries",
321
- "entries must be a list of {user, role}. Send [] to un-share, which is how "
322
- "revoking is expressed")
323
- _entries_or_400(session, entries)
324
- out = shares.set_grants(kind, oid, entries, owner=rec["owner"] or session.uname,
325
- st=session.runtime)
326
- _notify_new_grantees(session, kind, oid, before=rec["entries"], after=out.get("entries") or [])
327
- return out
328
-
329
-
330
- def _notify_new_grantees(session, kind, oid, before, after):
331
- """⭐⭐ W32-T28 (owner item 18's last clause, contract C3) — tell the RECEIVER, in their Inbox.
332
-
333
- Owner item 18 ends *"being shared a database notifies the receiver"*. Until now sharing was
334
- silent: the grant landed in a rail section the receiver had to notice on their own, which is
335
- why "I shared it with you" and "I never saw it" were both true.
336
-
337
- WRITTEN ON THE SHARE, NEVER POLLED. `/notifications` re-evaluates view-ALERTS on read
338
- because an alert is a live question about rows; a share is an EVENT that happened once, and
339
- polling for it would mean re-deriving "was this new?" on every inbox open the diff below
340
- only exists here, at the moment the set changes.
341
-
342
- ⚠ ONLY THE NEWLY ADDED. `PUT` REPLACES the whole entry set (revoking is expressed by absence),
343
- so every save re-sends everyone who was already there. Diffing against `before` is what stops
344
- a rename or a role change from ringing the bell for people whose access did not change.
345
- `*` IS NOT NOTIFIED: there is no user to name, and minting one notification per account in
346
- the tenant on a single click is a broadcast nobody asked for. The rail still shows it.
347
- ⚠ IT NEVER RAISES. A notification that fails must not fail the share that triggered it — the
348
- grant is the user's actual intent, and `core.alerts.notify` writes with `flush='async'`.
349
- """
350
- try:
351
- was = {e.get("user") for e in (before or ()) if isinstance(e, dict)}
352
- fresh = [str(e.get("user")) for e in (after or ())
353
- if isinstance(e, dict) and e.get("user") not in was
354
- and e.get("user") != shares.EVERYONE]
355
- if not fresh:
356
- return
357
- import core.alerts as alerts
358
-
359
- label, route, view_id = _object_ref(session, kind, oid)
360
- if not route:
361
- # NO ROUTE, NO NOTIFICATION — the receiver would get a row that opens nothing, and
362
- # `notification_view` would have to invent a target. Silence is the honest answer
363
- # here; the rail still shows the grant under "Shared with me".
364
- return
365
- sharer = str(session.user.get("name") or session.uname)
366
- for user in fresh:
367
- # THE SHAPE IS `routes_alerts.notification_view`'s SHARE BRANCH, and the two must
368
- # agree or the Inbox row is unclickable: `topic` selects the branch and `key` becomes
369
- # `alertId`, which that branch reads as the id to open. Both constants are IMPORTED
370
- # from there rather than typed again — one vocabulary, one owner.
371
- # ⭐⭐ W33-T28 (`ASK C-14`, answered) `actor` IS THE SENDER, AND IT IS THE ONLY WAY
372
- # THE INBOX CAN NAME ONE. An alert and an automation have no person behind them and
373
- # are honestly named by their machine; a SHARE has a real person, and only this call
374
- # site knows who. It is passed as its OWN field rather than recovered from the
375
- # `detail` prose below: a sender parsed out of "<name> shared this with you" breaks
376
- # the first time the sentence is reworded, silently, in the header
377
- # [[grep-output-is-not-source]]. The prose stays as the body; this is the From.
378
- alerts.notify(user, label, topic=_SHARE_TOPIC, key=route, row_id=view_id,
379
- detail=f"{sharer} shared this with you", actor=sharer,
380
- st=session.runtime)
381
- except Exception: # noqa: BLE001
382
- return
383
-
384
-
385
- def _object_ref(session, kind, oid):
386
- """`(label, route, view_id)` — what to CALL the shared thing, and where it OPENS.
387
-
388
- ⛔ THE ROUTE IS RESOLVED HERE, NOT SHAPED IN THE CONSUMER, AND THE FIRST VERSION GOT IT
389
- WRONG: it put the raw `oid` in the notification's key, so a shared VIEW produced
390
- `target: {module: "database", id: "view_42"}` — an instruction to open a database named
391
- `view_42`. It read perfectly in the payload and would have opened nothing. **A view is not
392
- addressable on its own; it is a SELECTION inside a topic's grid**, so the pair is what has to
393
- travel. Caught by looking at the notification the driver actually produced, not by reading
394
- the code back.
395
-
396
- `label` never falls back to a raw id. A notification headed `ut_leads_3f2a` tells the
397
- receiver nothing they can act on, and the id is already in the target.
398
- An unresolvable object answers `route=None`, and the caller then sends NOTHING rather than
399
- a row that opens nowhere.
400
- """
401
- try:
402
- if kind == "field":
403
- # ⭐⭐ W38-T16 A COLUMN IS NOT ADDRESSABLE ON ITS OWN, exactly as a view is not: it
404
- # is a column INSIDE a database, so the target that travels is the DATABASE. Without
405
- # this branch the function falls through to the view/folder loop, finds nothing,
406
- # answers `route=None` — and `_notify_new_grantees` returns EARLY. The grant lands and
407
- # the receiver is never told, which is the silent half of owner item 18 reopened one
408
- # kind over.
409
- from routes_alerts import route_for_topic
410
- table_key, field_key = shares.split_field_oid(oid)
411
- if not table_key:
412
- return ("A column", None, "")
413
- try:
414
- import core.shared_overlay as shared_overlay
415
- defn = (shared_overlay.fields(table_key, st=session.runtime) or {}).get(field_key)
416
- except Exception: # noqa: BLE001
417
- defn = None
418
- label = str((defn or {}).get("label") or "").strip() or field_key
419
- # ⚠ TWO SPELLINGS REACH THIS LINE AND ONE MAP ANSWERS BOTH. `shared_overlay` is keyed
420
- # by whatever the calling door already held: a `ut_*` database uses its bare key,
421
- # while a registry topic uses `<topic>_table_workspace` (`product_data.TABLE_KEY`).
422
- # `route_for_topic` speaks the GRID SCOPE vocabulary (`customer`, not
423
- # `customer_data`), so the suffix comes off before it is asked — rather than a second
424
- # route table being written here, which is how the two come apart.
425
- _WS = "_table_workspace"
426
- scope = table_key[:-len(_WS)] if table_key.endswith(_WS) else table_key
427
- return (label, route_for_topic(scope) or None, "")
428
- if kind == "database":
429
- import core.user_tables as ut
430
- defn = (ut.all_defs(st=session.runtime) or {}).get(str(oid)) or {}
431
- # A user table IS its own route key in both vocabularies (`route_for_topic`).
432
- return (str(defn.get("label") or "").strip() or "A database", str(oid), "")
433
- import core.table_store as table_store
434
- from routes_alerts import route_for_topic
435
- for topic in _topics(session):
436
- ops = table_store.make(f"{topic}_table_workspace", st=session.runtime)
437
- hit = ops.find_view(oid) if kind == "view" else ops.find_folder(oid)
438
- if not hit:
439
- continue
440
- route = route_for_topic(topic)
441
- if not route:
442
- break
443
- row = hit[1] if len(hit) > 1 else {}
444
- name = str((row or {}).get("name") or "").strip()
445
- # ⚠ Only a VIEW carries a selection. A folder is a rail grouping, so the target opens
446
- # the grid and stops there rather than naming a view the receiver did not get.
447
- return (name or ("A view" if kind == "view" else "A folder"),
448
- route, str(oid) if kind == "view" else "")
449
- except Exception: # noqa: BLE001
450
- pass
451
- return ({"view": "A view", "folder": "A folder",
452
- "field": "A column"}.get(kind, "An item"), None, "")
 
 
 
 
 
 
 
1
+ """routes_shares.py — the manage-access surface (wave 20, owner ruling R10, contract C-SHARE).
2
+
3
+ GET /api/v1/share/{kind}/{oid} -> {owner, entries:[{user,role}], mayAdminister, people}
4
+ PUT /api/v1/share/{kind}/{oid} <- {entries:[{user,role}]} (REPLACES the set)
5
+ GET /api/v1/share/mine -> {view:[id], folder:[id], database:[id]}
6
+
7
+ `kind` ∈ view | folder | database | field. Roles are `view` | `edit` — the same two words the
8
+ view rail already speaks, now extended to folders, databases and COLUMNS so there is ONE
9
+ vocabulary in the UI (R10: "the same picker views use").
10
+
11
+ ⭐⭐ **W38-T16 — `field` IS THE FOURTH KIND, AND ITS `oid` IS TOPIC-QUALIFIED: `"<table_key>:<field_key>"`**
12
+ (`shares.field_oid`). A bare column key repeats across databases — `notes` exists on a dozen —
13
+ so a grant stored under one would admit the grantee to every `notes` column in the tenant at
14
+ once. ⛔ THREE functions in this file branch on kind and ALL THREE need the new one, which is not
15
+ obvious because only two of them fail loudly: `_owns_object` (without it a column's own creator
16
+ is 404'd trying to share the thing they just made) and `_object_ref` (without it `route` is None,
17
+ `_notify_new_grantees` returns early, and the grantee is **granted and never told** — owner item
18
+ 18's silent half, reopened one kind over). `_can_see_object` stays deliberately CLOSED for
19
+ anything that is not a view.
20
+
21
+ ⛔ **RE-SHARING IS THE OWNER'S, AND THAT IS ENFORCED HERE, NOT IN THE CLIENT.** `PUT` requires
22
+ `shares.may_administer` (owner or admin). A collaborator with `edit` may change an object's
23
+ CONTENT and may not change who else can reach it — otherwise anyone you shared a view with could
24
+ widen it to everyone, or grant themselves ownership and lock you out. The client greys the editor
25
+ for non-administrators; that is a courtesy, and this check is the wall.
26
+
27
+ ⚠ **THE GRANT NEVER WIDENS PAST THE MODULE WALL — ON A GOVERNED MODULE.** `*` ("everyone") means
28
+ every account that can already open the surface: `require_session` plus the topic's own gate run
29
+ first, and for `customer_data` / `product_data` the receiver's own row scope and hidden-field
30
+ closure run BEFORE any foreign view is merged. Sharing there can only narrow-or-equal the set that
31
+ could already reach the data ([[aios-permissioning]]).
32
+
33
+ ⛔⛔ **AND THAT SENTENCE IS FALSE FOR `kind='database'`, WHICH IS WHY IT NOW SAYS "ON A GOVERNED
34
+ MODULE" (W32-T26, audit S-8).** `routes_admin._PERM_MODULES` is `("customer_data","product_data")`
35
+ and `_clean_perms` **400s** on anything else, so **no row filter and no hidden field can even be
36
+ DECLARED for a `ut_*` database** — `routes_tables.py` makes zero `perm_scope` calls and passes
37
+ `hidden_keys=frozenset()`. There is no module wall behind a user table for a grant to be bounded
38
+ by: **this registry IS the wall.** So a `database` grant is ALL-OR-NOTHING — every row, every
39
+ column — and an `*` database grant admits every account in the tenant to all of it.
40
+ That is a real capability, deliberately kept; what was wrong was a docstring promising a second
41
+ wall that does not exist for this kind. Scoping user tables is booked, not done
42
+ (`waves/wave32/sharing-audit.md` S-8).
43
+
44
+ ⚠ **TWO SYSTEMS ANSWER "IS THIS SHARED", AND THEY ARE NOT THE SAME ONE (audit S-4).** THIS
45
+ registry decides who appears in *"Shared with me"* and who may re-share. **`table_store.is_shared`
46
+ — the view's own `permissions` — is what actually decides who may OPEN a view.** A grant here
47
+ whose object is invisible under that one is a row in a list that opens a refusal, which is what
48
+ made item 18 worth auditing. `_entries_or_400` closes the common cause (a name nobody has), but
49
+ the two vocabularies are still two.
50
+ """
51
+ from fastapi import APIRouter, Body, Depends
52
+
53
+ import core.shares as shares
54
+ import core.users as users
55
+ from deps import Session, err, require_session
56
+ # ⭐ W32-T28 (C3) — the SHARE notification's topic word, imported from the module that CLASSIFIES
57
+ # it (`routes_alerts.notification_view`) rather than typed again here. The producer and the
58
+ # reader agreeing about one string is the whole difference between an Inbox row that opens the
59
+ # shared database and one that is quietly unclickable.
60
+ from routes_alerts import SHARE_TOPIC as _SHARE_TOPIC
61
+
62
+ router = APIRouter(prefix="/api/v1")
63
+
64
+
65
+ def _kind_or_400(raw):
66
+ try:
67
+ return shares._check_kind(raw)
68
+ except ValueError as e:
69
+ raise err(400, "bad_kind", str(e))
70
+
71
+
72
+ # ── ⭐⭐ WAVE 32 · T26 (owner item 18, ruling R12) — THE WALL THIS FILE SAID IT HAD ─────────────
73
+ #
74
+ # `put_share`'s comment used to justify the first-claim rule with *"reaching this route at all
75
+ # means passing the surface's own wall"*. **There was no such wall.** `kind` and `oid` are free
76
+ # strings off the URL and the only dependency was `require_session`, so any signed-in account
77
+ # could `PUT` a grant on an id it had never seen. Because the 403 sat behind `if rec["owner"]`,
78
+ # an object with no grant record skipped the check entirely and the caller was stamped OWNER —
79
+ # sticky, so **the real creator was then refused on their own view, permanently.** Driven, not
80
+ # argued: `waves/wave32/sharing-audit.md` S-1 carries the four-step transcript.
81
+ #
82
+ # ⚠ AND IT WAS SILENT ON BOTH SIDES. The claimant does not even see the object in their own
83
+ # "Shared with me" (`shared_with` excludes what you own), so nothing appears anywhere until the
84
+ # victim next opens the dialog.
85
+
86
+ #: The built-in grid topics. A view or folder lives in `{topic}_table_workspace`, and the share
87
+ #: route is not told which topic — so resolving one means asking each.
88
+ _BUILTIN_TOPICS = ("customer", "product")
89
+
90
+
91
+ def _topics(session):
92
+ """Every topic whose workspace could hold a view or folder for this tenant.
93
+
94
+ ⚠ `all_defs`, never `all_tables` — the latter is the whole 28.6 MB row payload (~703 ms on
95
+ tenant #0) to answer a question about KEYS (D-185).
96
+ """
97
+ try:
98
+ import core.user_tables as ut
99
+ return (*_BUILTIN_TOPICS, *(ut.all_defs(st=session.runtime) or {}))
100
+ except Exception: # noqa: BLE001
101
+ return _BUILTIN_TOPICS
102
+
103
+
104
+ def _owns_object(session, kind, oid):
105
+ """May this caller CLAIM an object that has no grant record yet — i.e. do they own it?
106
+
107
+ ⛔ THIS GUARDS THE CLAIM, NOT THE READ, AND THAT IS DELIBERATE. Resolving a view means asking
108
+ each topic's workspace in turn, which is N store reads; making every share call pay that
109
+ would put a loop on a route the manage-access dialog opens. The dangerous path is the one
110
+ where a caller is about to be stamped OWNER of something nobody owns — so the resolution runs
111
+ exactly there, and the common path (a record exists, `may_administer` decides) is untouched.
112
+ """
113
+ if session.admin:
114
+ return True
115
+ if kind == "field":
116
+ # ⭐⭐ W38-T16 — A COLUMN'S OWNER IS ITS `createdBy`, WHICH THE CREATE DOOR ALREADY STAMPS
117
+ # (`routes_tables.patch_shared_cell`) and the DELETE door already reads as its wall (R8 /
118
+ # D-172: creator-or-admin). Read from the same place by all three, so a column cannot be
119
+ # deletable by one person and shareable by another.
120
+ # ⚠ THIS BRANCH IS NOT OPTIONAL AND ITS ABSENCE FAILS SILENTLY IN THE WORST DIRECTION:
121
+ # a brand-new column has no grant record, so `put_share` falls to this predicate — and
122
+ # without it the column's own creator is answered `404 no_object` on the first attempt to
123
+ # share the thing they just made.
124
+ table_key, field_key = shares.split_field_oid(oid)
125
+ if not table_key:
126
+ return False
127
+ try:
128
+ import core.shared_overlay as shared_overlay
129
+ defn = (shared_overlay.fields(table_key, st=session.runtime) or {}).get(field_key)
130
+ except Exception: # noqa: BLE001
131
+ return False
132
+ return bool(defn) and str(defn.get("createdBy") or "") == str(session.uname)
133
+ if kind == "database":
134
+ # ⚠ `may_open` is THE resolver for a user table (its own docstring says so) and already
135
+ # admits creator, admin, or a `database` grantee. Re-implementing "who owns a table"
136
+ # here would be the second definition this wave keeps finding.
137
+ try:
138
+ import core.user_tables as ut
139
+ return bool(ut.may_open(oid, session.uname, is_admin=session.admin,
140
+ st=session.runtime))
141
+ except Exception: # noqa: BLE001
142
+ return False
143
+ try:
144
+ import core.table_store as table_store
145
+ except Exception: # noqa: BLE001
146
+ return False
147
+ for topic in _topics(session):
148
+ try:
149
+ ops = table_store.make(f"{topic}_table_workspace", st=session.runtime)
150
+ hit = ops.find_view(oid) if kind == "view" else ops.find_folder(oid)
151
+ except Exception: # noqa: BLE001
152
+ continue
153
+ if hit:
154
+ # `find_view`/`find_folder` answer `(owner_username, …)`. The claim belongs to the
155
+ # person whose personal stratum holds it — anybody else reaching this line is
156
+ # exactly the case S-1 describes.
157
+ return str(hit[0]) == str(session.uname)
158
+ return False
159
+
160
+
161
+ def _can_see_object(session, kind, oid):
162
+ """May this caller READ an object's grant list — i.e. can they reach the object at all?
163
+
164
+ ⛔⛔ THIS IS DELIBERATELY WIDER THAN {@link _owns_object}, AND CONFLATING THE TWO IS A
165
+ REGRESSION I SHIPPED AND CAUGHT. The first version of T26 guarded BOTH doors with the
166
+ ownership test, which reads sensibly and is wrong for the read, because **`find_view` searches
167
+ PERSONAL STRATA ONLY** (its own docstring says so). So a view living in alice's stratum with
168
+ `permissions.edit = "collaborative"` and no grant record yet — a view bob **can open and edit
169
+ in the grid** — answered `404` when bob opened its manage-access dialog. Measured before
170
+ fixing: `table_store._may_see(view, "bob") is True` while `GET /share/view/vc` said
171
+ `404 no_object`.
172
+ ⚠ THAT IS THE AUDIT'S OWN S-4 BITING THE AUDIT'S OWN FIX: two systems answer "is this shared",
173
+ and the wall consulted the grant registry (system A) plus stratum ownership, never the view's
174
+ `permissions` (system B) — which is the one that actually decides who may OPEN it.
175
+ ⚠ And it hides the ANSWER, not just the editor. `ViewSidebar`'s Share row is deliberately not
176
+ gated on edit rights because *"hiding the row from everyone else would hide the ANSWER too —
177
+ 'who has this?' is a fair question for anyone the view was shared with"*. A 404 there tells a
178
+ legitimate collaborator their view does not exist.
179
+
180
+ ⛔ THE CLAIM KEEPS THE NARROW TEST. Being able to SEE an object must not let you become its
181
+ owner — that is S-1, and widening this predicate onto `put_share` would re-open it.
182
+ """
183
+ if _owns_object(session, kind, oid):
184
+ return True
185
+ if kind != "view":
186
+ # A folder carries no per-object visibility flag of its own, and a database's `may_open`
187
+ # (inside `_owns_object`) already admits grantees. Nothing wider to ask.
188
+ # ⭐ W38-T16 — AND `field` KEEPS THIS CLOSED, DELIBERATELY. A grantee never reaches here:
189
+ # `get_share` tests `role is None` first and a grant answers a role, so the only caller
190
+ # left is an account with no relationship to the column at all. Widening it would let any
191
+ # signed-in session enumerate who holds which column on a database they cannot open.
192
+ return False
193
+ try:
194
+ import core.table_store as table_store
195
+ for topic in _topics(session):
196
+ hit = table_store.make(f"{topic}_table_workspace", st=session.runtime).find_view(oid)
197
+ if hit:
198
+ return bool(table_store._may_see(hit[1] if len(hit) > 1 else {},
199
+ session.uname, is_admin=session.admin))
200
+ except Exception: # noqa: BLE001
201
+ return False
202
+ return False
203
+
204
+
205
+ def _entries_or_400(session, entries):
206
+ """Validate a grant list against the tenant's REAL, ACTIVE accounts — and refuse BY NAME.
207
+
208
+ ⛔ `core.shares._clean_entries` silently drops junk, and its docstring argues that correctly:
209
+ a UI mid-save must not lose the whole list to one malformed row. **But it validates the SHAPE
210
+ of a string and the role word — never that the user EXISTS, is ACTIVE, or is in this tenant**,
211
+ so a typo'd name is stored, reported as a successful save, and never reaches anybody. The
212
+ sharer believes the person has access. That is item 18's plain reading.
213
+ ⚠ The correct population is computed THREE FUNCTIONS BELOW and served to the picker
214
+ (`_people`). One route, two populations, and the write door was the permissive one.
215
+ ⚠ `*` (everyone) is not a user and is admitted deliberately — it is R10's vocabulary for
216
+ "every account that can already open the surface".
217
+ """
218
+ known = {p["username"].strip().lower() for p in _people(session.tenant)}
219
+ unknown = []
220
+ for e in entries or ():
221
+ if not isinstance(e, dict):
222
+ continue
223
+ user = str(e.get("user") or "").strip().lower()
224
+ if user and user != shares.EVERYONE and user not in known:
225
+ unknown.append(user)
226
+ if unknown:
227
+ raise err(400, "unknown_people",
228
+ "no active account in this workspace is named "
229
+ + ", ".join(sorted(set(unknown)))
230
+ + ". Nothing was shared. Pick people from the list rather than typing a name.")
231
+
232
+
233
+ @router.get("/share/mine")
234
+ def my_shares(session: Session = Depends(require_session)):
235
+ """Everything shared WITH me, by kind — the "Shared with me" rail section (R10).
236
+
237
+ Registered before `/share/{kind}/{oid}` so the literal path wins the match; FastAPI resolves
238
+ in declaration order and `mine` would otherwise be read as a `kind`, answering 400 for a URL
239
+ that is not malformed at all.
240
+ """
241
+ return shares.shared_with(session.uname, st=session.runtime)
242
+
243
+
244
+ @router.get("/share/{kind}/{oid}")
245
+ def get_share(kind: str, oid: str, session: Session = Depends(require_session)):
246
+ kind = _kind_or_400(kind)
247
+ rec = shares.grants(kind, oid, st=session.runtime)
248
+ role = shares.role_for(kind, oid, session.uname, is_admin=session.admin, st=session.runtime)
249
+ may_admin = shares.may_administer(kind, oid, session.uname, is_admin=session.admin,
250
+ st=session.runtime)
251
+ # W39-T29 the dialog opens with GET before its first PUT. Until that PUT exists the grant
252
+ # registry has no owner to return from `may_administer`, even though the same caller may safely
253
+ # claim their own object through the PUT path below. Reflect that exact claim predicate here:
254
+ # a Member who owns an unshared View receives the people picker; a collaborator still does not.
255
+ if not rec["owner"] and not may_admin and _owns_object(session, kind, oid):
256
+ may_admin = True
257
+ # W32-T26 (audit S-3) A STRANGER LEARNS NOTHING. This route used to answer for ANY id:
258
+ # who owns it, everyone it is granted to, and the tenant's whole username↔name directory —
259
+ # to any signed-in session, about objects it cannot open. Now a caller with no role on an
260
+ # object must prove they can reach it, and gets a 404 otherwise: the same answer a
261
+ # non-existent id gives, so the route cannot be used to probe which ids are real.
262
+ # ⚠ `role is None` is the cheap pre-test, so the N-topic resolution below runs only for a
263
+ # caller who has no relationship with the object at all.
264
+ if role is None and not _can_see_object(session, kind, oid):
265
+ raise err(404, "no_object", "no such item, or it is not shared with this account")
266
+ return {
267
+ **rec,
268
+ "role": role,
269
+ "mayAdminister": may_admin,
270
+ # WAVE 21 (C1 identity fix): grant entries BIND on USERNAMES, so the picker must carry
271
+ # them. `assignable_people` serves bare display names because `user`-kind CELLS store
272
+ # display names that list's shape cannot change without migrating cell values — so
273
+ # this route serves objects of its own. Existing grants that were written as lowercased
274
+ # display names are normalised by the wave-21 cleanup script.
275
+ # ⭐ W32-T26 (audit S-3) — the roster is the EDITOR's data, so it rides only for a caller
276
+ # who may open the editor. A read-only grantee gets the grant list (their fair question is
277
+ # "who else has this?") and not a directory of every account in the workspace.
278
+ "people": _people(session.tenant) if may_admin else [],
279
+ }
280
+
281
+
282
+ def _people(tenant):
283
+ """[{username, name}] for this tenant same population as `assignable_people`, with the
284
+ BINDING identity alongside the display one."""
285
+ try:
286
+ reg = users.registry() or {}
287
+ except Exception:
288
+ return []
289
+ want = str(tenant or '').strip().lower()
290
+ out = []
291
+ for uname, u in reg.items():
292
+ if not isinstance(u, dict) or u.get('active') is False:
293
+ continue
294
+ if want and str(u.get('tenant') or 'royal-imports').strip().lower() != want:
295
+ continue
296
+ out.append({"username": str(uname), "name": str(u.get('name') or uname)})
297
+ return sorted(out, key=lambda p: p["name"].lower())
298
+
299
+
300
+ @router.put("/share/{kind}/{oid}")
301
+ def put_share(kind: str, oid: str, body: dict = Body(default=None),
302
+ session: Session = Depends(require_session)):
303
+ kind = _kind_or_400(kind)
304
+ body = body or {}
305
+ rec = shares.grants(kind, oid, st=session.runtime)
306
+ # An object with NO grant record yet has no owner — the first person to share it claims it.
307
+ # That is safe because reaching this route at all means passing the surface's own wall, and
308
+ # the alternative (refusing until somebody seeds an owner) would make a brand-new folder
309
+ # unshareable by the person who just made it.
310
+ if rec["owner"]:
311
+ if not shares.may_administer(kind, oid, session.uname, is_admin=session.admin,
312
+ st=session.runtime):
313
+ raise err(403, "not_owner",
314
+ "only the owner of this item (or an administrator) can change who it is "
315
+ "shared with")
316
+ # ⛔⛔ W32-T26 (audit S-1) — THE CLAIM NOW HAS A PRECONDITION. An object with no grant record
317
+ # is still claimed by the first person to share it that rule is right, and refusing until
318
+ # somebody seeds an owner would make a brand-new folder unshareable by the person who just
319
+ # made it. What was missing is the half the old comment ASSERTED and the code never did: the
320
+ # claimant has to be able to reach the object. Without this, any signed-in account could
321
+ # stamp itself owner of an id it had never seen and lock the real creator out for good.
322
+ elif not _owns_object(session, kind, oid):
323
+ raise err(404, "no_object", "no such item, or it is not shared with this account")
324
+ entries = body.get("entries")
325
+ if not isinstance(entries, list):
326
+ raise err(400, "bad_entries",
327
+ "entries must be a list of {user, role}. Send [] to un-share, which is how "
328
+ "revoking is expressed")
329
+ _entries_or_400(session, entries)
330
+ out = shares.set_grants(kind, oid, entries, owner=rec["owner"] or session.uname,
331
+ st=session.runtime)
332
+ _notify_new_grantees(session, kind, oid, before=rec["entries"], after=out.get("entries") or [])
333
+ return out
334
+
335
+
336
+ def _notify_new_grantees(session, kind, oid, before, after):
337
+ """⭐⭐ W32-T28 (owner item 18's last clause, contract C3) tell the RECEIVER, in their Inbox.
338
+
339
+ Owner item 18 ends *"being shared a database notifies the receiver"*. Until now sharing was
340
+ silent: the grant landed in a rail section the receiver had to notice on their own, which is
341
+ why "I shared it with you" and "I never saw it" were both true.
342
+
343
+ WRITTEN ON THE SHARE, NEVER POLLED. `/notifications` re-evaluates view-ALERTS on read
344
+ because an alert is a live question about rows; a share is an EVENT that happened once, and
345
+ polling for it would mean re-deriving "was this new?" on every inbox open the diff below
346
+ only exists here, at the moment the set changes.
347
+
348
+ ONLY THE NEWLY ADDED. `PUT` REPLACES the whole entry set (revoking is expressed by absence),
349
+ so every save re-sends everyone who was already there. Diffing against `before` is what stops
350
+ a rename or a role change from ringing the bell for people whose access did not change.
351
+ `*` IS NOT NOTIFIED: there is no user to name, and minting one notification per account in
352
+ the tenant on a single click is a broadcast nobody asked for. The rail still shows it.
353
+ IT NEVER RAISES. A notification that fails must not fail the share that triggered it — the
354
+ grant is the user's actual intent, and `core.alerts.notify` writes with `flush='async'`.
355
+ """
356
+ try:
357
+ was = {e.get("user") for e in (before or ()) if isinstance(e, dict)}
358
+ fresh = [str(e.get("user")) for e in (after or ())
359
+ if isinstance(e, dict) and e.get("user") not in was
360
+ and e.get("user") != shares.EVERYONE]
361
+ if not fresh:
362
+ return
363
+ import core.alerts as alerts
364
+
365
+ label, route, view_id = _object_ref(session, kind, oid)
366
+ if not route:
367
+ # NO ROUTE, NO NOTIFICATION the receiver would get a row that opens nothing, and
368
+ # `notification_view` would have to invent a target. Silence is the honest answer
369
+ # here; the rail still shows the grant under "Shared with me".
370
+ return
371
+ sharer = str(session.user.get("name") or session.uname)
372
+ for user in fresh:
373
+ # THE SHAPE IS `routes_alerts.notification_view`'s SHARE BRANCH, and the two must
374
+ # agree or the Inbox row is unclickable: `topic` selects the branch and `key` becomes
375
+ # `alertId`, which that branch reads as the id to open. Both constants are IMPORTED
376
+ # from there rather than typed again one vocabulary, one owner.
377
+ # ⭐⭐ W33-T28 (`ASK C-14`, answered) `actor` IS THE SENDER, AND IT IS THE ONLY WAY
378
+ # THE INBOX CAN NAME ONE. An alert and an automation have no person behind them and
379
+ # are honestly named by their machine; a SHARE has a real person, and only this call
380
+ # site knows who. ⛔ It is passed as its OWN field rather than recovered from the
381
+ # `detail` prose below: a sender parsed out of "<name> shared this with you" breaks
382
+ # the first time the sentence is reworded, silently, in the header
383
+ # [[grep-output-is-not-source]]. The prose stays as the body; this is the From.
384
+ alerts.notify(user, label, topic=_SHARE_TOPIC, key=route, row_id=view_id,
385
+ detail=f"{sharer} shared this with you", actor=sharer,
386
+ st=session.runtime)
387
+ except Exception: # noqa: BLE001
388
+ return
389
+
390
+
391
+ def _object_ref(session, kind, oid):
392
+ """`(label, route, view_id)` what to CALL the shared thing, and where it OPENS.
393
+
394
+ THE ROUTE IS RESOLVED HERE, NOT SHAPED IN THE CONSUMER, AND THE FIRST VERSION GOT IT
395
+ WRONG: it put the raw `oid` in the notification's key, so a shared VIEW produced
396
+ `target: {module: "database", id: "view_42"}` an instruction to open a database named
397
+ `view_42`. It read perfectly in the payload and would have opened nothing. **A view is not
398
+ addressable on its own; it is a SELECTION inside a topic's grid**, so the pair is what has to
399
+ travel. Caught by looking at the notification the driver actually produced, not by reading
400
+ the code back.
401
+
402
+ `label` never falls back to a raw id. A notification headed `ut_leads_3f2a` tells the
403
+ receiver nothing they can act on, and the id is already in the target.
404
+ An unresolvable object answers `route=None`, and the caller then sends NOTHING rather than
405
+ a row that opens nowhere.
406
+ """
407
+ try:
408
+ if kind == "field":
409
+ # ⭐⭐ W38-T16 — A COLUMN IS NOT ADDRESSABLE ON ITS OWN, exactly as a view is not: it
410
+ # is a column INSIDE a database, so the target that travels is the DATABASE. Without
411
+ # this branch the function falls through to the view/folder loop, finds nothing,
412
+ # answers `route=None` — and `_notify_new_grantees` returns EARLY. The grant lands and
413
+ # the receiver is never told, which is the silent half of owner item 18 reopened one
414
+ # kind over.
415
+ from routes_alerts import route_for_topic
416
+ table_key, field_key = shares.split_field_oid(oid)
417
+ if not table_key:
418
+ return ("A column", None, "")
419
+ try:
420
+ import core.shared_overlay as shared_overlay
421
+ defn = (shared_overlay.fields(table_key, st=session.runtime) or {}).get(field_key)
422
+ except Exception: # noqa: BLE001
423
+ defn = None
424
+ label = str((defn or {}).get("label") or "").strip() or field_key
425
+ # TWO SPELLINGS REACH THIS LINE AND ONE MAP ANSWERS BOTH. `shared_overlay` is keyed
426
+ # by whatever the calling door already held: a `ut_*` database uses its bare key,
427
+ # while a registry topic uses `<topic>_table_workspace` (`product_data.TABLE_KEY`).
428
+ # `route_for_topic` speaks the GRID SCOPE vocabulary (`customer`, not
429
+ # `customer_data`), so the suffix comes off before it is asked — rather than a second
430
+ # route table being written here, which is how the two come apart.
431
+ _WS = "_table_workspace"
432
+ scope = table_key[:-len(_WS)] if table_key.endswith(_WS) else table_key
433
+ return (label, route_for_topic(scope) or None, "")
434
+ if kind == "database":
435
+ import core.user_tables as ut
436
+ defn = (ut.all_defs(st=session.runtime) or {}).get(str(oid)) or {}
437
+ # A user table IS its own route key in both vocabularies (`route_for_topic`).
438
+ return (str(defn.get("label") or "").strip() or "A database", str(oid), "")
439
+ import core.table_store as table_store
440
+ from routes_alerts import route_for_topic
441
+ for topic in _topics(session):
442
+ ops = table_store.make(f"{topic}_table_workspace", st=session.runtime)
443
+ hit = ops.find_view(oid) if kind == "view" else ops.find_folder(oid)
444
+ if not hit:
445
+ continue
446
+ route = route_for_topic(topic)
447
+ if not route:
448
+ break
449
+ row = hit[1] if len(hit) > 1 else {}
450
+ name = str((row or {}).get("name") or "").strip()
451
+ # Only a VIEW carries a selection. A folder is a rail grouping, so the target opens
452
+ # the grid and stops there rather than naming a view the receiver did not get.
453
+ return (name or ("A view" if kind == "view" else "A folder"),
454
+ route, str(oid) if kind == "view" else "")
455
+ except Exception: # noqa: BLE001
456
+ pass
457
+ return ({"view": "A view", "folder": "A folder",
458
+ "field": "A column"}.get(kind, "An item"), None, "")
api/routes_slack.py CHANGED
@@ -1,772 +1,772 @@
1
- """routes_slack.py — MANAGE AGENT: a bot per Slack channel, walled by the engine that walls a
2
- person. (wave 33, owner item 10 · `W33-T38` / `W33-T39` / `W33-T67`.)
3
-
4
- Owner, verbatim (2026-08-14): *"Expand Slack we need to be able to configure an AI bot PER channel
5
- whose permissions we can toggle based on Fields etc of the databases, exactly like how we toggle
6
- permissioning per user."*
7
-
8
- ────────────────────────────────────────────────────────────────────────────────────────────────
9
- ⛔⛔ D-84 SAID THIS WAS "adapted from OpenTag" AND THAT PREMISE IS FALSE (PRD amendment A1).
10
-
11
- `PENDING.md` D-84 has named OpenTag as the model for per-Slack-channel permissioning since
12
- 2026-08-08. **OpenTag has no permission model to adapt.** `app/runtime-host.ts` is
13
- `identifyUser: () => OPENTAG_SERVICE_USER` — ONE constant identity for every request — and its own
14
- `sender-context.ts` says the Slack sender is *"informational only… not gating access."* Its
15
- authorization ceiling is whole-MCP-server, decided once at process boot from env vars. It is a
16
- live specimen of identity that LOOKS like a permission and is not.
17
-
18
- `core/perm_scope.py` is already strictly more expressive: per database, per row, per FIELD, per
19
- principal. So item 10 is OUR engine projected onto a CHANNEL principal — see `agent_principal`
20
- below, which is the whole projection and is nine lines. Study:
21
- `.claude/wiki/research/opentag-adoption.md` (licence gate: MIT, PASSED).
22
-
23
- What OpenTag genuinely contributes is the WRITE-APPROVAL interceptor (`W33-T67`, `approval_card`
24
- below) and nothing else. Its TRANSPORT is AVOID wholesale — hosted CopilotKit Intelligence, a
25
- second Node process, a persistent outbound socket and each tenant's Slack credentials held by a
26
- third party, against ONE process on the HF free tier. The door here is `routes_forms.py`'s proven
27
- unauthenticated-public shape plus Slack's signing-secret check.
28
-
29
- ────────────────────────────────────────────────────────────────────────────────────────────────
30
- ⚠ A CHANNEL IS A PRINCIPAL, NOT A PERSON. The wall binds to the Slack conversation id, so every
31
- human in that channel reads through the same rules. That is the honest reading of "a bot per
32
- channel" and the pane says it out loud, because the alternative is an admin assuming per-person
33
- scoping from a screen that looks exactly like the per-person one.
34
-
35
- ⚠ AND THE IDENTITY IS THE CHANNEL ID, NEVER ITS NAME. A channel can be renamed; its id cannot.
36
- A wall bound to `#sales` silently re-points the day somebody renames the channel.
37
- """
38
- import hmac
39
- import json
40
- import os
41
- import secrets
42
- import time
43
- from datetime import datetime, timezone
44
-
45
- from fastapi import APIRouter, Body, Depends, Request
46
-
47
- from deps import Session, err, require_session
48
- from routes_admin import admin_gate
49
-
50
- router = APIRouter(prefix="/api/v1")
51
-
52
- #: The tenant's channel agents: `{id: record}`. A per-tenant bucket, so it rides
53
- #: `runtime.store_key`'s prefix and never lands in tenant #0's namespace.
54
- AGENTS_KEY = "slack_agents"
55
-
56
- #: Pending mutations awaiting a human's click (`W33-T67`). Server-only, per tenant.
57
- PENDING_KEY = "slack_pending_writes"
58
-
59
- #: How long an approval card stays clickable. An approval is a statement about the world as it was
60
- #: when the card was rendered; an hour later the values it names may no longer be the values it
61
- #: would write. Expiring is the honest behaviour, and an expired card says so rather than 403ing.
62
- APPROVAL_TTL_S = 15 * 60
63
-
64
- #: Slack rejects a replayed request older than 5 minutes; so do we, before any signature work.
65
- SLACK_MAX_SKEW_S = 60 * 5
66
-
67
- MAX_BODY_BYTES = 64 * 1024
68
-
69
- #: ⛔ ROW-CAPPED BECAUSE AN APPROVER WHO CANNOT READ THE CARD CANNOT APPROVE IT. Copied
70
- #: deliberately from OpenTag's `agent/write_confirmation.py`, which caps for the same reason:
71
- #: past a certain length Slack collapses a message behind "Show more", and a human clicking
72
- #: Approve on a card whose tail they never saw is worse than no card at all.
73
- APPROVAL_MAX_ROWS = 12
74
-
75
-
76
- # ── the tenant's agent records ───────────────────────────────────────────────────────────────
77
- def _agents(rt):
78
- """`{id: record}` for one tenant. `{}` on any failure — an unreadable bucket must degrade to
79
- "this tenant has no agents", never to a 500 on the settings pane."""
80
- try:
81
- found = rt.get(AGENTS_KEY) or {}
82
- except Exception: # noqa: BLE001
83
- return {}
84
- return found if isinstance(found, dict) else {}
85
-
86
-
87
- def _clean_agent(raw, prior=None):
88
- """One stored record, built KEY BY KEY.
89
-
90
- ⛔ Never `dict(raw)` and never `**raw`. This record is the SUBJECT of a permission decision;
91
- a client-supplied key landing in it is a client-supplied input to `perm_scope`. The same rule
92
- `routes_forms._public_form` follows on the way out, applied here on the way in.
93
- """
94
- prior = prior if isinstance(prior, dict) else {}
95
- out = {
96
- "id": str(prior.get("id") or raw.get("id") or ""),
97
- "channel": str(raw.get("channel") or prior.get("channel") or "").strip()[:64],
98
- "channelName": str(raw.get("channelName") or prior.get("channelName") or "").strip()[:80],
99
- "label": " ".join(str(raw.get("label") or prior.get("label") or "").split())[:80],
100
- "active": bool(raw.get("active", prior.get("active", True))),
101
- # ⛔ DEFAULT TRUE, AND `is not False` RATHER THAN TRUTHINESS. A record written before this
102
- # key existed must read as "approval required": the one direction a default here is
103
- # allowed to be wrong in is the safe one, because an unattended write cannot be undone by
104
- # revoking the bot afterwards [[default-must-pass-its-own-guard]].
105
- "writeApproval": raw.get("writeApproval", prior.get("writeApproval", True)) is not False,
106
- # The permission block. Same shape, same validator, same engine as a user's.
107
- "perms": prior.get("perms") if isinstance(prior.get("perms"), dict) else {},
108
- "perms_v": int(prior.get("perms_v") or 0),
109
- "createdBy": str(prior.get("createdBy") or raw.get("createdBy") or ""),
110
- "createdAt": str(prior.get("createdAt") or raw.get("createdAt") or ""),
111
- }
112
- return out
113
-
114
-
115
- def _put_agent(session, agent_id, mutate):
116
- """Read-modify-write ONE agent under the tenant's bucket, synchronously.
117
-
118
- `flush="sync"` because the client re-reads the list immediately after every write here, and an
119
- async flush would let the refetch win the race and paint the pre-write state — the
120
- [[refetch-eats-its-own-write]] shape.
121
- """
122
- def _set(cur):
123
- cur = dict(cur or {})
124
- rec = cur.get(agent_id) if isinstance(cur.get(agent_id), dict) else None
125
- nxt = mutate(rec)
126
- if nxt is None:
127
- cur.pop(agent_id, None)
128
- else:
129
- cur[agent_id] = nxt
130
- return cur
131
-
132
- session.runtime.update(AGENTS_KEY, _set, flush="sync")
133
-
134
-
135
- # ── ⭐⭐ THE PROJECTION — THE WHOLE OF "the same engine that permissions a user" ───────────────
136
- def agent_principal(agent):
137
- """A channel agent, as a PRINCIPAL `core/perm_scope` already understands.
138
-
139
- ⭐⭐ THIS IS THE ENTIRE POINT OF THE TICKET AND IT IS NINE LINES. `perm_scope` takes a user
140
- RECORD — a plain dict with `perms`, `perms_v` and `role` — not a User object and not a session.
141
- So a channel agent carrying a `perms` block simply IS such a record, and `may_access`,
142
- `visible_fields`, `hidden_keys` and `apply_row_scope` work on it unmodified. There is no second
143
- engine, no parallel vocabulary, and nothing about a channel to keep in step with anything about
144
- a person. That is what the owner's *"exactly like how we toggle permissioning per user"* means
145
- when it is true in code rather than in appearance.
146
-
147
- ⛔ `role` IS HARDCODED NON-ADMIN AND MUST STAY SO. `perm_scope.may_access` returns True
148
- unconditionally for `role == 'admin'` — the break-glass clause that keeps the owner out of a
149
- locked store. A bot has no such emergency and no hands: an agent record that could carry
150
- `role: 'admin'` would be one stored key away from bypassing every wall on this page. The key is
151
- not copied from the record; it is written here, every time.
152
-
153
- ⛔ AND `perms_v` IS FORCED TO THE CURRENT VERSION, which makes an UNDECLARED database DENY
154
- (C-PERM amendment 4). Without it a fresh agent would fall through to `perms.may_open`'s legacy
155
- grant wall and read `modules: 'all'`-shaped defaults — i.e. a brand-new bot would start with
156
- access to everything. Fail-closed from the first byte.
157
- """
158
- import core.perm_scope as perm_scope
159
- return {
160
- "username": f"slack:{(agent or {}).get('channel') or '?'}",
161
- "role": "agent",
162
- "perms": (agent or {}).get("perms") or {},
163
- "perms_v": perm_scope.PERMS_VERSION,
164
- # No `bus`, no `agent`: a channel is not a business unit and not a salesperson. Absent
165
- # rather than 'all' — `perms.allowed_bu_labels` narrows on absence, which is the direction
166
- # this must fail in.
167
- }
168
-
169
-
170
- def agent_may_read(agent, module):
171
- """May this channel open `module` at all?"""
172
- import core.perm_scope as perm_scope
173
- if not (agent or {}).get("active", True):
174
- return False
175
- return bool(perm_scope.may_access(agent_principal(agent), module))
176
-
177
-
178
- def agent_visible_fields(agent, fields, module):
179
- """`fields` minus this channel's hidden closure — the SAME transitive closure a person gets,
180
- so a formula over a hidden column cannot leak it back through arithmetic."""
181
- import core.perm_scope as perm_scope
182
- return perm_scope.visible_fields(fields, agent_principal(agent), module)
183
-
184
-
185
- def agent_rows(agent, rows, module, fields, ctx=None):
186
- """The rows this channel may receive — `permits()`, so an unanswerable permanent filter DENIES
187
- rather than being ignored."""
188
- import core.perm_scope as perm_scope
189
- p = agent_principal(agent)
190
- hide = perm_scope.hidden_keys(p, module, fields)
191
- kept = perm_scope.apply_row_scope(rows, p, module, fields, ctx)
192
- # Both wires, never one: the field list and the row payload are separate, and stripping only
193
- # the first leaves the value sitting in the second where anyone can read it.
194
- return [perm_scope.strip_row(r, hide) for r in kept]
195
-
196
-
197
- # ── the credential ───────────────────────────────────────────────────────────────────────────
198
- def slack_creds(rt):
199
- """This TENANT's Slack credentials from its own keychain, or None.
200
-
201
- ⛔ NO ENVIRONMENT FALLBACK, DELIBERATELY, AND THIS IS D-202's LESSON APPLIED BEFORE IT
202
- HAPPENS AGAIN. Meta Ads shipped loading end-to-end and was still not a connector, because
203
- `keychain.meta_creds` had no caller and the token came from the owner's `.env` — making it a
204
- tenant-#0 FACT indistinguishable from a screenshot. `keychain.meta_creds`' own docstring spells
205
- out the rule: handing the environment's credential to a tenant whose admin has not stored one
206
- is the leak the resolver exists to prevent. So a tenant with no entry gets None and the pane
207
- SAYS so.
208
-
209
- ⚠ Reads through `list_entries` + `read_fields` (both public) rather than `keychain._first_creds`
210
- (private, and in a file outside this lane's fence). Same deterministic rule: entries are
211
- id-sorted, so "first" is stable across reads rather than dict-order luck.
212
- """
213
- try:
214
- import core.keychain as keychain
215
- for row in keychain.list_entries(rt):
216
- if str(row.get("type") or "") != "slack":
217
- continue
218
- fields = keychain.read_fields(rt, row.get("id"))
219
- if isinstance(fields, dict) and fields.get("signing_secret"):
220
- return fields
221
- except Exception: # noqa: BLE001
222
- # A locked or unreadable keychain reads as "not configured" for the PANE, which is honest;
223
- # the signature check below fails closed regardless, so this cannot widen anything.
224
- return None
225
- return None
226
-
227
-
228
- # ── the authenticated doors (the Manage agent surface) ───────────────────────────────────────
229
- def _summary(agent, modules):
230
- """One sentence per agent for the list row — computed here, because the alternative is one
231
- round trip per row to fill one cell (the same reason `AdminUser.access` exists)."""
232
- # ⭐ W36-T22 / C2 — every listed database is governed now; the `enforced` flag it used to
233
- # filter on is DELETED, and `m.get("enforced")` would have gone silently falsy here and
234
- # summarised every agent as "" (no databases at all).
235
- governed = list(modules)
236
- perms = agent.get("perms") or {}
237
- open_n = sum(1 for m in governed if (perms.get(m["key"]) or {}).get("access"))
238
- if not governed:
239
- return ""
240
- if open_n == 0:
241
- return "No access"
242
- restricted = sum(1 for m in governed
243
- if (perms.get(m["key"]) or {}).get("access")
244
- and ((perms.get(m["key"]) or {}).get("filter")
245
- or (perms.get(m["key"]) or {}).get("hiddenFields")))
246
- head = (f"All {open_n} database{'' if open_n == 1 else 's'}" if open_n == len(governed)
247
- else f"{open_n} of {len(governed)} databases")
248
- return f"{head}, {restricted} restricted" if restricted else head
249
-
250
-
251
- @router.get("/agents")
252
- def list_channel_agents(session: Session = Depends(admin_gate)):
253
- """This tenant's channel agents, plus whether Slack is reachable at all."""
254
- import routes_admin
255
- modules = routes_admin._perm_modules(session)
256
- rows = []
257
- for aid, a in sorted(_agents(session.runtime).items()):
258
- if not isinstance(a, dict):
259
- continue
260
- rows.append({"id": aid, "channel": a.get("channel") or "",
261
- "channelName": a.get("channelName") or "",
262
- "label": a.get("label") or "", "active": a.get("active", True) is not False,
263
- "writeApproval": a.get("writeApproval", True) is not False,
264
- "summary": _summary(a, modules)})
265
- return {"agents": rows, "configured": slack_creds(session.runtime) is not None}
266
-
267
-
268
- @router.post("/agents")
269
- def create_channel_agent(body: dict = Body(default=None),
270
- session: Session = Depends(admin_gate)):
271
- body = body if isinstance(body, dict) else {}
272
- channel = str(body.get("channel") or "").strip()
273
- if not channel:
274
- raise err(400, "no_channel", "a Slack channel ID is required")
275
- existing = _agents(session.runtime)
276
- # ⚠ ONE AGENT PER CHANNEL. Two records for one conversation would mean two walls, and nothing
277
- # anywhere decides which one applies — the [[one-question-two-normalizers]] shape, in the one
278
- # place where the two answers are "may read" and "may not".
279
- for a in existing.values():
280
- if isinstance(a, dict) and str(a.get("channel") or "") == channel:
281
- raise err(409, "channel_taken",
282
- "that channel already has an agent — edit it instead of adding a second")
283
- aid = secrets.token_urlsafe(9)
284
- rec = _clean_agent(dict(body, id=aid, createdBy=session.uname,
285
- createdAt=datetime.now(timezone.utc).isoformat(timespec="seconds")))
286
- _put_agent(session, aid, lambda _prior: rec)
287
- fresh = _agents(session.runtime).get(aid)
288
- if not isinstance(fresh, dict):
289
- # The store took the write and did not record it. A 200 here would tell an administrator
290
- # the agent exists when it does not.
291
- raise err(503, "store_unavailable", "the agent was NOT created")
292
- return {"agent": {"id": aid, "channel": rec["channel"], "channelName": rec["channelName"],
293
- "label": rec["label"], "active": rec["active"],
294
- "writeApproval": rec["writeApproval"], "summary": "No access"}}
295
-
296
-
297
- @router.get("/agents/{agent_id}")
298
- def get_channel_agent(agent_id: str, session: Session = Depends(admin_gate)):
299
- """The agent PLUS the permission envelope — the SAME shape `get_perms` answers with.
300
-
301
- ⭐ ONE PAYLOAD SHAPE FOR TWO ROOMS. `permsModel.parsePermsPayload` parses this, and
302
- `settings/ModulePermsList` renders it, because they are literally the same client code. A
303
- second envelope here would mean a second parser and a second set of defaults within a wave.
304
- """
305
- import routes_admin
306
- a = _agents(session.runtime).get(str(agent_id))
307
- if not isinstance(a, dict):
308
- raise err(404, "no_such_agent", "no agent with that id")
309
- modules = routes_admin._perm_modules(session)
310
- # ⭐⭐ W36-T22 / C2 — EVERY listed database, `ut_*` included. ⛔ THIS IS WHY THE FLAG'S
311
- # DELETION IS NOT A ONE-FILE CHANGE: `m.get("enforced")` on a row that no longer carries the
312
- # key is None, so this list would have been EMPTY and the channel perms editor would have
313
- # governed ZERO databases while looking entirely correct. A Slack channel is a principal of
314
- # the SAME wall (`verify_perm_scope` section H) and gets the same catalogue.
315
- governed = [m["key"] for m in modules]
316
- stored = a.get("perms") or {}
317
- principal = agent_principal(a)
318
- import core.perm_scope as perm_scope
319
- perms_out = {}
320
- for k in governed:
321
- e = stored.get(k)
322
- # ⛔ W36-T22 — `may_read`, the same evaluator the read door uses, for the reason spelled
323
- # out at `routes_admin.get_perms`. ⚠ It answers DIFFERENTLY here and correctly so: a
324
- # channel agent is not a person with a share, so a `ut_*` key it has not been granted
325
- # defaults CLOSED — which is the fail-closed direction for a bot, and the same answer the
326
- # table routes would give it.
327
- perms_out[k] = e if isinstance(e, dict) else {
328
- "access": bool(perm_scope.may_read(principal, k, st=session.runtime)),
329
- "filter": None, "hiddenFields": []}
330
- return {"id": str(agent_id), "channel": a.get("channel") or "",
331
- "channelName": a.get("channelName") or "", "label": a.get("label") or "",
332
- "active": a.get("active", True) is not False,
333
- "writeApproval": a.get("writeApproval", True) is not False,
334
- "perms": perms_out,
335
- # ⛔ ALWAYS THE CURRENT VERSION, because `agent_principal` forces it: the editor must
336
- # render "undeclared = denied", not the legacy-grant reading it would show at 0.
337
- "perms_v": perm_scope.PERMS_VERSION,
338
- # A bot is NEVER an admin — see `agent_principal`. Sent so the editor never paints
339
- # the "Everything (admin)" state for a principal that cannot have it.
340
- "is_admin": False,
341
- "modules": modules,
342
- "fields_by_module": {k: routes_admin._module_fields(k, session=session)
343
- for k in governed}}
344
-
345
-
346
- @router.put("/agents/{agent_id}/perms")
347
- def put_channel_agent_perms(agent_id: str, body: dict = Body(default=None),
348
- session: Session = Depends(admin_gate)):
349
- """Replace this agent's permission block wholesale — the same validator a user's block gets."""
350
- import routes_admin
351
- body = body if isinstance(body, dict) else {}
352
- if not isinstance(_agents(session.runtime).get(str(agent_id)), dict):
353
- raise err(404, "no_such_agent", "no agent with that id")
354
- if "perms" not in body:
355
- raise err(400, "empty_patch", "no perms to save")
356
- mods = routes_admin._perm_modules(session)
357
- # ⭐ THE SAME `_clean_perms`, NOT A COPY OF IT. It refuses a filter this module cannot
358
- # evaluate, refuses a hiddenFields key that names nothing, and refuses a BU condition the
359
- # pushdown cannot read. Every one of those is exactly as true for a channel as for a person,
360
- # and a second validator here is a second place for one of them to go missing.
361
- # ⚠ CORRECTED W36-T22: this list used to end "refuses an unenforced `ut_*` key". That refusal
362
- # is DELETED — W36-T21 armed the wall over every database, so there is no unenforced key left
363
- # to refuse, and a comment naming a rule that no longer exists is how the next wave re-derives
364
- # it ([[two-gates-can-assert-opposite-things]]).
365
- cleaned = routes_admin._clean_perms(
366
- body.get("perms"),
367
- governed_keys={m["key"] for m in mods}, session=session) or {}
368
-
369
- import core.perm_scope as perm_scope
370
-
371
- def _mut(prior):
372
- if not isinstance(prior, dict):
373
- return None
374
- nxt = dict(prior)
375
- nxt["perms"] = cleaned
376
- nxt["perms_v"] = perm_scope.PERMS_VERSION
377
- return nxt
378
-
379
- _put_agent(session, str(agent_id), _mut)
380
- fresh = _agents(session.runtime).get(str(agent_id)) or {}
381
- if (fresh.get("perms") or {}) != cleaned:
382
- # The one direction this must never fail in: telling an administrator the wall is up when
383
- # it is not.
384
- raise err(503, "store_unavailable", "the permissions were not saved — the agent is UNCHANGED")
385
- return {"id": str(agent_id), "perms": fresh.get("perms") or {},
386
- "perms_v": int(fresh.get("perms_v") or 0)}
387
-
388
-
389
- @router.patch("/agents/{agent_id}")
390
- def patch_channel_agent(agent_id: str, body: dict = Body(default=None),
391
- session: Session = Depends(admin_gate)):
392
- body = body if isinstance(body, dict) else {}
393
- if not isinstance(_agents(session.runtime).get(str(agent_id)), dict):
394
- raise err(404, "no_such_agent", "no agent with that id")
395
- #: ⛔ `perms` IS NOT PATCHABLE HERE. It has its own door with its own validator; accepting it
396
- #: on the metadata route would be a second, unvalidated way to write the wall.
397
- allowed = {k: v for k, v in body.items()
398
- if k in ("label", "active", "writeApproval", "channelName")}
399
- if not allowed:
400
- raise err(400, "empty_patch", "nothing to change")
401
-
402
- def _mut(prior):
403
- return _clean_agent(allowed, prior=prior) if isinstance(prior, dict) else None
404
-
405
- _put_agent(session, str(agent_id), _mut)
406
- import routes_admin
407
- a = _agents(session.runtime).get(str(agent_id)) or {}
408
- return {"agent": {"id": str(agent_id), "channel": a.get("channel") or "",
409
- "channelName": a.get("channelName") or "", "label": a.get("label") or "",
410
- "active": a.get("active", True) is not False,
411
- "writeApproval": a.get("writeApproval", True) is not False,
412
- "summary": _summary(a, routes_admin._perm_modules(session))}}
413
-
414
-
415
- @router.delete("/agents/{agent_id}")
416
- def delete_channel_agent(agent_id: str, session: Session = Depends(admin_gate)):
417
- if not isinstance(_agents(session.runtime).get(str(agent_id)), dict):
418
- raise err(404, "no_such_agent", "no agent with that id")
419
- _put_agent(session, str(agent_id), lambda _prior: None)
420
- if isinstance(_agents(session.runtime).get(str(agent_id)), dict):
421
- raise err(503, "store_unavailable", "the agent was NOT removed")
422
- return {"ok": True}
423
-
424
-
425
- # ── ⭐ W33-T67: THE WRITE-APPROVAL GATE ───────────────────────────────────────────────────────
426
- def _is_empty(value):
427
- """Is this value ABSENT, as opposed to falsy?
428
-
429
- ⛔ COPIED EXACTLY FROM OpenTag's `agent/write_confirmation.py`, and the exactness is the point:
430
- `0` and `False` are VALUES a human must see on the card. Setting a price to 0 or a flag to
431
- False is precisely the kind of write somebody needs to approve, and a naive `if not value`
432
- drops both rows silently — the approver then reads a card that does not describe the write
433
- they are approving. The MIT licence gate for this borrowing is recorded in
434
- `.claude/wiki/research/opentag-adoption.md`.
435
- """
436
- return value is None or (isinstance(value, str) and value.strip() == "")
437
-
438
-
439
- def approval_card(action, values, agent=None):
440
- """The card a human reads before a bot writes. `{action, rows, truncated, note}`.
441
-
442
- ⛔ ROW-CAPPED, and the cap is a SAFETY property rather than a layout one: past a certain
443
- length Slack collapses a message behind "Show more", and **an approver who cannot read the
444
- card cannot approve it.** When rows are dropped the card SAYS how many — a truncation nobody
445
- was told about is the violation, not the truncation (wave-30 R6's second sentence).
446
- """
447
- rows = [{"field": str(k), "value": v}
448
- for k, v in (values or {}).items() if not _is_empty(v)]
449
- rows.sort(key=lambda r: r["field"])
450
- shown, dropped = rows[:APPROVAL_MAX_ROWS], max(0, len(rows) - APPROVAL_MAX_ROWS)
451
- return {
452
- "action": str(action or "")[:80],
453
- "rows": shown,
454
- "truncated": dropped,
455
- "note": (f"{dropped} more field(s) not shown — open the record to review them before "
456
- f"approving." if dropped else ""),
457
- "agent": (agent or {}).get("label") or (agent or {}).get("channel") or "",
458
- }
459
-
460
-
461
- def intercept_write(agent, action, values, stash):
462
- """THE INTERCEPTOR. Returns `("commit", None)` or `("await_approval", card)`.
463
-
464
- ⭐ The one thing worth taking from OpenTag: a MUTATING tool call is halted, rendered as a card
465
- naming the action and its values, and committed only on an explicit human accept. Everything
466
- else in that repo — the transport, the runtime, the identity model — is AVOID (amendment A1).
467
-
468
- ⛔ FAIL-CLOSED ON A MISSING FLAG: `is not False`, so a record written before `writeApproval`
469
- existed requires approval. The unsafe direction here is not recoverable — revoking a bot does
470
- not un-write what it wrote.
471
- """
472
- if (agent or {}).get("writeApproval", True) is not False:
473
- card = approval_card(action, values, agent)
474
- stash(card)
475
- return "await_approval", card
476
- return "commit", None
477
-
478
-
479
- def _pending(rt):
480
- try:
481
- found = rt.get(PENDING_KEY) or {}
482
- except Exception: # noqa: BLE001
483
- return {}
484
- return found if isinstance(found, dict) else {}
485
-
486
-
487
- def stash_pending(rt, agent, action, values, now=None):
488
- """Store one awaiting-approval mutation and return its token.
489
-
490
- The token is what rides the card's button, so it is `secrets.token_urlsafe` and not the
491
- action's id: a guessable value on an unauthenticated door is a way to approve somebody else's
492
- write. Server-only bucket, per tenant, exactly as `routes_forms.TOKENS_KEY` is.
493
- """
494
- token = secrets.token_urlsafe(18)
495
- row = {"at": float(now if now is not None else time.time()),
496
- "channel": str((agent or {}).get("channel") or ""),
497
- "action": str(action or "")[:80],
498
- "values": values if isinstance(values, dict) else {}}
499
- rt.update(PENDING_KEY, lambda cur: {**(cur or {}), token: row}, flush="sync")
500
- return token
501
-
502
-
503
- #: `{action kind: fn(rt, row) -> None}`. Registered by whatever can actually perform a mutation.
504
- #:
505
- #: ⛔⛔ EMPTY IN PRODUCTION TODAY, AND SAYING SO IS THE POINT. `commit_pending` below is the ONE
506
- #: place an approved write may be performed, so it is the place a negative control can bite — but
507
- #: a gate is only as honest as what it guards. Until something registers here, an approved card
508
- #: performs NOTHING and `commit_pending` returns `no_committer` rather than pretending. That is a
509
- #: declared absence, not a silent one [[flag-shipped-without-its-writer]].
510
- _COMMITTERS = {}
511
-
512
-
513
- def register_committer(kind, fn):
514
- """Declare who may perform an approved mutation of `kind`. Idempotent by key."""
515
- _COMMITTERS[str(kind)] = fn
516
-
517
-
518
- def commit_pending(rt, token, approved, now=None):
519
- """Resolve ONE approval card. Returns one of
520
- `expired` · `unknown` · `rejected` · `no_committer` · `committed` · `failed`.
521
-
522
- ⛔⛔ THIS IS THE WRITE GATE, AND IT IS A SEPARATE FUNCTION FROM THE ROUTE ON PURPOSE. T67's
523
- `done-when` asks for an NC proving a REJECTED card writes nothing. A control aimed at the
524
- route could not bite while the route had no write in it at all — it would stay green with the
525
- guard deleted, because there would be nothing for the guard to be protecting anything from
526
- ([[gate-can-report-green-on-nothing]], and a verifier caught exactly that on this ticket's
527
- first draft). Putting the commit behind ONE named seam gives the control a real subject: the
528
- gate registers a recorder, and `rejected` must leave it untouched while `approve` must call it
529
- exactly once.
530
-
531
- ⛔ THE TOKEN IS CONSUMED BEFORE EITHER BRANCH, and before any committer runs. A card that
532
- survives its own click is a replayable write, and "approve twice" must never mean "write
533
- twice". The delete is `flush="sync"` for the same reason.
534
- """
535
- now = float(now if now is not None else time.time())
536
- row = _pending(rt).get(str(token))
537
- if not isinstance(row, dict):
538
- return "unknown"
539
- # Consume FIRST — before the TTL verdict, before the approve/reject branch, before any write.
540
- rt.update(PENDING_KEY,
541
- lambda cur: {k: v for k, v in (cur or {}).items() if k != str(token)},
542
- flush="sync")
543
- if now - float(row.get("at") or 0) > APPROVAL_TTL_S:
544
- return "expired"
545
- if not approved:
546
- # ⛔ NOTHING BELOW THIS LINE RUNS FOR A REJECTED CARD. The whole gate is this return.
547
- return "rejected"
548
- fn = _COMMITTERS.get(str(row.get("action") or ""))
549
- if fn is None:
550
- return "no_committer"
551
- try:
552
- fn(rt, row)
553
- except Exception: # noqa: BLE001
554
- # A committer that raises must not read as a commit. The card is already consumed, so the
555
- # honest report is that it failed, and the human asks again.
556
- return "failed"
557
- return "committed"
558
-
559
-
560
- # ── the UNAUTHENTICATED Slack door ───────────────────────────────────────────────────────────
561
- #: Sliding window, keyed on the SLACK TEAM, never the client IP.
562
- #:
563
- #: ⛔ WHY NOT THE IP, WHICH IS WHAT `routes_forms` DOES. Every Slack event arrives from Slack's own
564
- #: infrastructure, so an IP window is one shared bucket for every workspace on the platform: one
565
- #: busy tenant would spend the allowance for all of them, and the symptom would be another
566
- #: tenant's bot going quiet. The team id is the closest thing to the actual noisy party.
567
- #: ⚠ It is attacker-CONTROLLED before the signature is checked, so the window is applied AFTER
568
- #: verification, never before — an unsigned request is refused on cost grounds anyway (a signature
569
- #: check is a single HMAC).
570
- RATE_WINDOW_S, RATE_PER_WINDOW = 60, 60
571
- _HITS: dict = {}
572
-
573
-
574
- def _rate_ok(key, now):
575
- seen = [t for t in _HITS.get(key, ()) if now - t < RATE_WINDOW_S]
576
- seen.append(now)
577
- _HITS[key] = seen
578
- if len(_HITS) > 4096:
579
- for k in [k for k, v in _HITS.items() if not v or now - v[-1] > RATE_WINDOW_S]:
580
- _HITS.pop(k, None)
581
- return len(seen) <= RATE_PER_WINDOW
582
-
583
-
584
- async def _raw_body(request):
585
- """The body as RAW BYTES, capped.
586
-
587
- ⛔⛔ RAW, NOT PARSED, AND THAT IS NOT A STYLE CHOICE. Slack signs the literal string
588
- `v0:{timestamp}:{body}` byte for byte. `routes_forms._bounded_body` — the function this is
589
- otherwise copied from — returns only the parsed dict and DISCARDS the bytes, so a verbatim
590
- copy of it could not verify a Slack signature at all: re-serialising the dict changes
591
- whitespace and key order and the HMAC no longer matches. The bug would look like "Slack
592
- signatures are always invalid", which is indistinguishable from a wrong secret.
593
-
594
- ⚠ And it streams rather than calling `request.body()`: FastAPI would otherwise have buffered
595
- the whole body before the handler's first line, so a content-length check inside the handler
596
- caps nothing (`routes_forms`' own scar, recorded in its docstring).
597
- """
598
- size, chunks = 0, []
599
- async for chunk in request.stream():
600
- size += len(chunk)
601
- if size > MAX_BODY_BYTES:
602
- raise err(413, "body_too_large", "that request is too large")
603
- chunks.append(chunk)
604
- return b"".join(chunks)
605
-
606
-
607
- def _refuse():
608
- """ONE refusal for every reason. No oracle.
609
-
610
- A wrong signature, a stale timestamp, an unknown team, an unconfigured tenant and a channel
611
- with no agent all answer identically — otherwise the door tells an unauthenticated caller
612
- which tenants exist and which channels are configured, which is the question it was built to
613
- not answer. `routes_forms._refuse` takes the same posture for the same reason.
614
- """
615
- return err(403, "bad_request", "that request could not be verified")
616
-
617
-
618
- def _verify(secret, timestamp, raw, signature, now):
619
- """Slack's v0 signature, with the replay window checked FIRST."""
620
- try:
621
- ts = int(str(timestamp or "0"))
622
- except ValueError:
623
- return False
624
- if abs(now - ts) > SLACK_MAX_SKEW_S:
625
- return False
626
- base = b"v0:" + str(ts).encode("ascii") + b":" + (raw or b"")
627
- import hashlib
628
- expected = "v0=" + hmac.new(str(secret).encode("utf-8"), base, hashlib.sha256).hexdigest()
629
- return hmac.compare_digest(expected, str(signature or ""))
630
-
631
-
632
- def _resolve_tenant(raw, timestamp, signature, now):
633
- """`(slug, runtime, creds)` for the tenant whose signing secret verifies this request.
634
-
635
- ⛔ THE TENANT IS DECIDED BY THE SIGNATURE, NEVER BY THE PAYLOAD. A body-supplied `team_id`
636
- would let an unauthenticated caller name the tenant it wants to be — the exact widening
637
- `deps._user_for`'s tenant-equality check closes on the authenticated side. So every configured
638
- tenant's secret is tried and the one that VERIFIES names the tenant.
639
-
640
- ⚠ NO EARLY `break` ON A FAILED CANDIDATE and no per-tenant error: the loop's cost must not
641
- depend on which tenant matched. `routes_forms._resolve` walks tenants the same way, for the
642
- same reason — a public door has no session to ask.
643
- """
644
- from harness import runtime as _rt
645
- hit = None
646
- try:
647
- slugs = list(_rt.known_tenants())
648
- except Exception: # noqa: BLE001
649
- return None, None, None
650
- for slug in slugs:
651
- try:
652
- rt = _rt.get_runtime(slug)
653
- except Exception: # noqa: BLE001
654
- continue
655
- creds = slack_creds(rt)
656
- if not creds:
657
- continue
658
- if _verify(creds.get("signing_secret"), timestamp, raw, signature, now) and hit is None:
659
- hit = (slug, rt, creds)
660
- return hit if hit else (None, None, None)
661
-
662
-
663
- @router.post("/slack/events")
664
- async def slack_events(request: Request):
665
- """Slack's Events API. UNAUTHENTICATED BY CONSTRUCTION — no `Depends(require_session)`.
666
-
667
- ⭐ Built like `routes_forms.py`'s public door and not like OpenTag's: one process, no vendor,
668
- no second runtime, no persistent outbound socket, and this tenant's Slack secret never leaves
669
- this deployment (amendment A1).
670
- """
671
- now = time.time()
672
- raw = await _raw_body(request)
673
- ts = request.headers.get("x-slack-request-timestamp")
674
- sig = request.headers.get("x-slack-signature")
675
- slug, rt, _creds = _resolve_tenant(raw, ts, sig, now)
676
- if not rt:
677
- raise _refuse()
678
-
679
- try:
680
- payload = json.loads(raw or b"{}")
681
- except ValueError:
682
- raise _refuse()
683
- if not isinstance(payload, dict):
684
- raise _refuse()
685
-
686
- # Slack's one-time endpoint handshake. Answered ONLY after the signature verified — an
687
- # unsigned challenge echo would confirm the endpoint exists to anyone who probes it.
688
- if payload.get("type") == "url_verification":
689
- return {"challenge": str(payload.get("challenge") or "")[:512]}
690
-
691
- team = str(payload.get("team_id") or slug)
692
- if not _rate_ok(f"{slug}:{team}", now):
693
- raise err(429, "too_many_requests", "too many requests — wait a moment and try again")
694
-
695
- event = payload.get("event") if isinstance(payload.get("event"), dict) else {}
696
- channel = str(event.get("channel") or "")
697
- agent = None
698
- for a in _agents(rt).values():
699
- if isinstance(a, dict) and str(a.get("channel") or "") == channel:
700
- agent = a
701
- break
702
- # ⛔ NO AGENT, OR AN INACTIVE ONE, IS SILENCE — a 200 with no action. Not a 403: Slack retries
703
- # a non-2xx up to three times, so refusing here would turn "this channel is not configured"
704
- # into three refusals per message, and the retry storm would be the only visible symptom.
705
- # ⚠ And it must not say WHICH — a distinguishable answer tells an unauthenticated caller which
706
- # channels this workspace has configured.
707
- if not isinstance(agent, dict) or agent.get("active", True) is False:
708
- return {"ok": True}
709
-
710
- # ⛔⛔ THE ANSWERING HALF IS NOT BUILT YET, AND SAYING SO IS THE POINT. What ships here is the
711
- # DOOR and the WALL: a verified request, resolved to a tenant by signature, matched to a
712
- # channel agent whose `perms` block is a `perm_scope` principal (`agent_principal`). Composing
713
- # a reply means an LLM call on the cheap-first ladder plus the read path, and a half-built
714
- # answerer that returns something plausible is worse than one that returns nothing.
715
- # Booked rather than faked [[flag-shipped-without-its-writer]].
716
- return {"ok": True}
717
-
718
-
719
- @router.post("/slack/interact")
720
- async def slack_interact(request: Request):
721
- """The approval card's button (`W33-T67`). UNAUTHENTICATED, signature-verified, same shape."""
722
- now = time.time()
723
- raw = await _raw_body(request)
724
- slug, rt, _creds = _resolve_tenant(raw, request.headers.get("x-slack-request-timestamp"),
725
- request.headers.get("x-slack-signature"), now)
726
- if not rt:
727
- raise _refuse()
728
- # Slack posts interactions as `application/x-www-form-urlencoded` with a `payload=` field.
729
- from urllib.parse import parse_qs
730
- try:
731
- form = parse_qs(raw.decode("utf-8"))
732
- payload = json.loads((form.get("payload") or ["{}"])[0])
733
- except Exception: # noqa: BLE001
734
- raise _refuse()
735
- if not isinstance(payload, dict):
736
- raise _refuse()
737
- if not _rate_ok(f"{slug}:interact", now):
738
- raise err(429, "too_many_requests", "too many requests — wait a moment and try again")
739
-
740
- actions = payload.get("actions") if isinstance(payload.get("actions"), list) else []
741
- choice = (actions[0] if actions and isinstance(actions[0], dict) else {})
742
- token = str(choice.get("value") or "")
743
- # ⭐ ANYTHING THAT IS NOT LITERALLY "approve" IS A REJECT. Fail-closed on a malformed,
744
- # truncated or unknown action id: the direction that can be wrong here without anyone
745
- # noticing is the one that writes.
746
- approved = str(choice.get("action_id") or "") == "approve"
747
- # ⛔ THE ROUTE DOES NOT DECIDE — `commit_pending` does, and it is the ONE seam a negative
748
- # control can aim at. Consumption, the TTL, the reject branch and the committer lookup all
749
- # live there; this handler only turns its verdict into a sentence.
750
- verdict = commit_pending(rt, token, approved, now=now)
751
- return {"text": {
752
- # An unknown, already-used or expired token is not an error: a person clicking a stale
753
- # card should be told it is stale, not shown a failure.
754
- "unknown": "That approval is no longer available. Ask the bot again.",
755
- "expired": "That approval expired before it was answered. Ask the bot again.",
756
- "rejected": "Rejected. Nothing was changed.",
757
- # ⚠ HONEST, not reassuring: the answering half that would register a committer is booked,
758
- # so an approved card today performs nothing and says exactly that rather than "Approved."
759
- "no_committer": "Approved — but this workspace has nothing configured to carry it out yet.",
760
- "failed": "That change could not be completed. Nothing was saved; ask the bot again.",
761
- "committed": "Approved.",
762
- }.get(verdict, "That approval is no longer available. Ask the bot again.")}
763
-
764
-
765
- @router.get("/slack/health")
766
- def slack_health(session: Session = Depends(require_session)):
767
- """Is Slack configured for THIS tenant? Session-gated, and it answers about one tenant only —
768
- enumerating the others would answer a question about our customer list."""
769
- creds = slack_creds(session.runtime)
770
- return {"configured": creds is not None,
771
- "agents": len(_agents(session.runtime)),
772
- "hasBotToken": bool((creds or {}).get("bot_token"))}
 
1
+ """routes_slack.py — MANAGE AGENT: a bot per Slack channel, walled by the engine that walls a
2
+ person. (wave 33, owner item 10 · `W33-T38` / `W33-T39` / `W33-T67`.)
3
+
4
+ Owner, verbatim (2026-08-14): *"Expand Slack we need to be able to configure an AI bot PER channel
5
+ whose permissions we can toggle based on Fields etc of the databases, exactly like how we toggle
6
+ permissioning per user."*
7
+
8
+ ────────────────────────────────────────────────────────────────────────────────────────────────
9
+ ⛔⛔ D-84 SAID THIS WAS "adapted from OpenTag" AND THAT PREMISE IS FALSE (PRD amendment A1).
10
+
11
+ `PENDING.md` D-84 has named OpenTag as the model for per-Slack-channel permissioning since
12
+ 2026-08-08. **OpenTag has no permission model to adapt.** `app/runtime-host.ts` is
13
+ `identifyUser: () => OPENTAG_SERVICE_USER` — ONE constant identity for every request — and its own
14
+ `sender-context.ts` says the Slack sender is *"informational only… not gating access."* Its
15
+ authorization ceiling is whole-MCP-server, decided once at process boot from env vars. It is a
16
+ live specimen of identity that LOOKS like a permission and is not.
17
+
18
+ `core/perm_scope.py` is already strictly more expressive: per database, per row, per FIELD, per
19
+ principal. So item 10 is OUR engine projected onto a CHANNEL principal — see `agent_principal`
20
+ below, which is the whole projection and is nine lines. Study:
21
+ `.claude/wiki/research/opentag-adoption.md` (licence gate: MIT, PASSED).
22
+
23
+ What OpenTag genuinely contributes is the WRITE-APPROVAL interceptor (`W33-T67`, `approval_card`
24
+ below) and nothing else. Its TRANSPORT is AVOID wholesale — hosted CopilotKit Intelligence, a
25
+ second Node process, a persistent outbound socket and each tenant's Slack credentials held by a
26
+ third party, against ONE process on the HF free tier. The door here is `routes_forms.py`'s proven
27
+ unauthenticated-public shape plus Slack's signing-secret check.
28
+
29
+ ────────────────────────────────────────────────────────────────────────────────────────────────
30
+ ⚠ A CHANNEL IS A PRINCIPAL, NOT A PERSON. The wall binds to the Slack conversation id, so every
31
+ human in that channel reads through the same rules. That is the honest reading of "a bot per
32
+ channel" and the pane says it out loud, because the alternative is an admin assuming per-person
33
+ scoping from a screen that looks exactly like the per-person one.
34
+
35
+ ⚠ AND THE IDENTITY IS THE CHANNEL ID, NEVER ITS NAME. A channel can be renamed; its id cannot.
36
+ A wall bound to `#sales` silently re-points the day somebody renames the channel.
37
+ """
38
+ import hmac
39
+ import json
40
+ import os
41
+ import secrets
42
+ import time
43
+ from datetime import datetime, timezone
44
+
45
+ from fastapi import APIRouter, Body, Depends, Request
46
+
47
+ from deps import Session, err, require_session
48
+ from routes_admin import admin_gate
49
+
50
+ router = APIRouter(prefix="/api/v1")
51
+
52
+ #: The tenant's channel agents: `{id: record}`. A per-tenant bucket, so it rides
53
+ #: `runtime.store_key`'s prefix and never lands in tenant #0's namespace.
54
+ AGENTS_KEY = "slack_agents"
55
+
56
+ #: Pending mutations awaiting a human's click (`W33-T67`). Server-only, per tenant.
57
+ PENDING_KEY = "slack_pending_writes"
58
+
59
+ #: How long an approval card stays clickable. An approval is a statement about the world as it was
60
+ #: when the card was rendered; an hour later the values it names may no longer be the values it
61
+ #: would write. Expiring is the honest behaviour, and an expired card says so rather than 403ing.
62
+ APPROVAL_TTL_S = 15 * 60
63
+
64
+ #: Slack rejects a replayed request older than 5 minutes; so do we, before any signature work.
65
+ SLACK_MAX_SKEW_S = 60 * 5
66
+
67
+ MAX_BODY_BYTES = 64 * 1024
68
+
69
+ #: ⛔ ROW-CAPPED BECAUSE AN APPROVER WHO CANNOT READ THE CARD CANNOT APPROVE IT. Copied
70
+ #: deliberately from OpenTag's `agent/write_confirmation.py`, which caps for the same reason:
71
+ #: past a certain length Slack collapses a message behind "Show more", and a human clicking
72
+ #: Approve on a card whose tail they never saw is worse than no card at all.
73
+ APPROVAL_MAX_ROWS = 12
74
+
75
+
76
+ # ── the tenant's agent records ───────────────────────────────────────────────────────────────
77
+ def _agents(rt):
78
+ """`{id: record}` for one tenant. `{}` on any failure — an unreadable bucket must degrade to
79
+ "this tenant has no agents", never to a 500 on the settings pane."""
80
+ try:
81
+ found = rt.get(AGENTS_KEY) or {}
82
+ except Exception: # noqa: BLE001
83
+ return {}
84
+ return found if isinstance(found, dict) else {}
85
+
86
+
87
+ def _clean_agent(raw, prior=None):
88
+ """One stored record, built KEY BY KEY.
89
+
90
+ ⛔ Never `dict(raw)` and never `**raw`. This record is the SUBJECT of a permission decision;
91
+ a client-supplied key landing in it is a client-supplied input to `perm_scope`. The same rule
92
+ `routes_forms._public_form` follows on the way out, applied here on the way in.
93
+ """
94
+ prior = prior if isinstance(prior, dict) else {}
95
+ out = {
96
+ "id": str(prior.get("id") or raw.get("id") or ""),
97
+ "channel": str(raw.get("channel") or prior.get("channel") or "").strip()[:64],
98
+ "channelName": str(raw.get("channelName") or prior.get("channelName") or "").strip()[:80],
99
+ "label": " ".join(str(raw.get("label") or prior.get("label") or "").split())[:80],
100
+ "active": bool(raw.get("active", prior.get("active", True))),
101
+ # ⛔ DEFAULT TRUE, AND `is not False` RATHER THAN TRUTHINESS. A record written before this
102
+ # key existed must read as "approval required": the one direction a default here is
103
+ # allowed to be wrong in is the safe one, because an unattended write cannot be undone by
104
+ # revoking the bot afterwards [[default-must-pass-its-own-guard]].
105
+ "writeApproval": raw.get("writeApproval", prior.get("writeApproval", True)) is not False,
106
+ # The permission block. Same shape, same validator, same engine as a user's.
107
+ "perms": prior.get("perms") if isinstance(prior.get("perms"), dict) else {},
108
+ "perms_v": int(prior.get("perms_v") or 0),
109
+ "createdBy": str(prior.get("createdBy") or raw.get("createdBy") or ""),
110
+ "createdAt": str(prior.get("createdAt") or raw.get("createdAt") or ""),
111
+ }
112
+ return out
113
+
114
+
115
+ def _put_agent(session, agent_id, mutate):
116
+ """Read-modify-write ONE agent under the tenant's bucket, synchronously.
117
+
118
+ `flush="sync"` because the client re-reads the list immediately after every write here, and an
119
+ async flush would let the refetch win the race and paint the pre-write state — the
120
+ [[refetch-eats-its-own-write]] shape.
121
+ """
122
+ def _set(cur):
123
+ cur = dict(cur or {})
124
+ rec = cur.get(agent_id) if isinstance(cur.get(agent_id), dict) else None
125
+ nxt = mutate(rec)
126
+ if nxt is None:
127
+ cur.pop(agent_id, None)
128
+ else:
129
+ cur[agent_id] = nxt
130
+ return cur
131
+
132
+ session.runtime.update(AGENTS_KEY, _set, flush="sync")
133
+
134
+
135
+ # ── ⭐⭐ THE PROJECTION — THE WHOLE OF "the same engine that permissions a user" ───────────────
136
+ def agent_principal(agent):
137
+ """A channel agent, as a PRINCIPAL `core/perm_scope` already understands.
138
+
139
+ ⭐⭐ THIS IS THE ENTIRE POINT OF THE TICKET AND IT IS NINE LINES. `perm_scope` takes a user
140
+ RECORD — a plain dict with `perms`, `perms_v` and `role` — not a User object and not a session.
141
+ So a channel agent carrying a `perms` block simply IS such a record, and `may_access`,
142
+ `visible_fields`, `hidden_keys` and `apply_row_scope` work on it unmodified. There is no second
143
+ engine, no parallel vocabulary, and nothing about a channel to keep in step with anything about
144
+ a person. That is what the owner's *"exactly like how we toggle permissioning per user"* means
145
+ when it is true in code rather than in appearance.
146
+
147
+ ⛔ `role` IS HARDCODED NON-ADMIN AND MUST STAY SO. `perm_scope.may_access` returns True
148
+ unconditionally for `role == 'admin'` — the break-glass clause that keeps the owner out of a
149
+ locked store. A bot has no such emergency and no hands: an agent record that could carry
150
+ `role: 'admin'` would be one stored key away from bypassing every wall on this page. The key is
151
+ not copied from the record; it is written here, every time.
152
+
153
+ ⛔ AND `perms_v` IS FORCED TO THE CURRENT VERSION, which makes an UNDECLARED database DENY
154
+ (C-PERM amendment 4). Without it a fresh agent would fall through to `perms.may_open`'s legacy
155
+ grant wall and read `modules: 'all'`-shaped defaults — i.e. a brand-new bot would start with
156
+ access to everything. Fail-closed from the first byte.
157
+ """
158
+ import core.perm_scope as perm_scope
159
+ return {
160
+ "username": f"slack:{(agent or {}).get('channel') or '?'}",
161
+ "role": "agent",
162
+ "perms": (agent or {}).get("perms") or {},
163
+ "perms_v": perm_scope.PERMS_VERSION,
164
+ # No `bus`, no `agent`: a channel is not a business unit and not a salesperson. Absent
165
+ # rather than 'all' — `perms.allowed_bu_labels` narrows on absence, which is the direction
166
+ # this must fail in.
167
+ }
168
+
169
+
170
+ def agent_may_read(agent, module):
171
+ """May this channel open `module` at all?"""
172
+ import core.perm_scope as perm_scope
173
+ if not (agent or {}).get("active", True):
174
+ return False
175
+ return bool(perm_scope.may_access(agent_principal(agent), module))
176
+
177
+
178
+ def agent_visible_fields(agent, fields, module):
179
+ """`fields` minus this channel's hidden closure — the SAME transitive closure a person gets,
180
+ so a formula over a hidden column cannot leak it back through arithmetic."""
181
+ import core.perm_scope as perm_scope
182
+ return perm_scope.visible_fields(fields, agent_principal(agent), module)
183
+
184
+
185
+ def agent_rows(agent, rows, module, fields, ctx=None):
186
+ """The rows this channel may receive — `permits()`, so an unanswerable permanent filter DENIES
187
+ rather than being ignored."""
188
+ import core.perm_scope as perm_scope
189
+ p = agent_principal(agent)
190
+ hide = perm_scope.hidden_keys(p, module, fields)
191
+ kept = perm_scope.apply_row_scope(rows, p, module, fields, ctx)
192
+ # Both wires, never one: the field list and the row payload are separate, and stripping only
193
+ # the first leaves the value sitting in the second where anyone can read it.
194
+ return [perm_scope.strip_row(r, hide) for r in kept]
195
+
196
+
197
+ # ── the credential ───────────────────────────────────────────────────────────────────────────
198
+ def slack_creds(rt):
199
+ """This TENANT's Slack credentials from its own keychain, or None.
200
+
201
+ ⛔ NO ENVIRONMENT FALLBACK, DELIBERATELY, AND THIS IS D-202's LESSON APPLIED BEFORE IT
202
+ HAPPENS AGAIN. Meta Ads shipped loading end-to-end and was still not a connector, because
203
+ `keychain.meta_creds` had no caller and the token came from the owner's `.env` — making it a
204
+ tenant-#0 FACT indistinguishable from a screenshot. `keychain.meta_creds`' own docstring spells
205
+ out the rule: handing the environment's credential to a tenant whose admin has not stored one
206
+ is the leak the resolver exists to prevent. So a tenant with no entry gets None and the pane
207
+ SAYS so.
208
+
209
+ ⚠ Reads through `list_entries` + `read_fields` (both public) rather than `keychain._first_creds`
210
+ (private, and in a file outside this lane's fence). Same deterministic rule: entries are
211
+ id-sorted, so "first" is stable across reads rather than dict-order luck.
212
+ """
213
+ try:
214
+ import core.keychain as keychain
215
+ for row in keychain.list_entries(rt):
216
+ if str(row.get("type") or "") != "slack":
217
+ continue
218
+ fields = keychain.read_fields(rt, row.get("id"))
219
+ if isinstance(fields, dict) and fields.get("signing_secret"):
220
+ return fields
221
+ except Exception: # noqa: BLE001
222
+ # A locked or unreadable keychain reads as "not configured" for the PANE, which is honest;
223
+ # the signature check below fails closed regardless, so this cannot widen anything.
224
+ return None
225
+ return None
226
+
227
+
228
+ # ── the authenticated doors (the Manage agent surface) ───────────────────────────────────────
229
+ def _summary(agent, modules):
230
+ """One sentence per agent for the list row — computed here, because the alternative is one
231
+ round trip per row to fill one cell (the same reason `AdminUser.access` exists)."""
232
+ # ⭐ W36-T22 / C2 — every listed database is governed now; the `enforced` flag it used to
233
+ # filter on is DELETED, and `m.get("enforced")` would have gone silently falsy here and
234
+ # summarised every agent as "" (no databases at all).
235
+ governed = list(modules)
236
+ perms = agent.get("perms") or {}
237
+ open_n = sum(1 for m in governed if (perms.get(m["key"]) or {}).get("access"))
238
+ if not governed:
239
+ return ""
240
+ if open_n == 0:
241
+ return "No access"
242
+ restricted = sum(1 for m in governed
243
+ if (perms.get(m["key"]) or {}).get("access")
244
+ and ((perms.get(m["key"]) or {}).get("filter")
245
+ or (perms.get(m["key"]) or {}).get("hiddenFields")))
246
+ head = (f"All {open_n} database{'' if open_n == 1 else 's'}" if open_n == len(governed)
247
+ else f"{open_n} of {len(governed)} databases")
248
+ return f"{head}, {restricted} restricted" if restricted else head
249
+
250
+
251
+ @router.get("/agents")
252
+ def list_channel_agents(session: Session = Depends(admin_gate)):
253
+ """This tenant's channel agents, plus whether Slack is reachable at all."""
254
+ import routes_admin
255
+ modules = routes_admin._perm_modules(session)
256
+ rows = []
257
+ for aid, a in sorted(_agents(session.runtime).items()):
258
+ if not isinstance(a, dict):
259
+ continue
260
+ rows.append({"id": aid, "channel": a.get("channel") or "",
261
+ "channelName": a.get("channelName") or "",
262
+ "label": a.get("label") or "", "active": a.get("active", True) is not False,
263
+ "writeApproval": a.get("writeApproval", True) is not False,
264
+ "summary": _summary(a, modules)})
265
+ return {"agents": rows, "configured": slack_creds(session.runtime) is not None}
266
+
267
+
268
+ @router.post("/agents")
269
+ def create_channel_agent(body: dict = Body(default=None),
270
+ session: Session = Depends(admin_gate)):
271
+ body = body if isinstance(body, dict) else {}
272
+ channel = str(body.get("channel") or "").strip()
273
+ if not channel:
274
+ raise err(400, "no_channel", "a Slack channel ID is required")
275
+ existing = _agents(session.runtime)
276
+ # ⚠ ONE AGENT PER CHANNEL. Two records for one conversation would mean two walls, and nothing
277
+ # anywhere decides which one applies — the [[one-question-two-normalizers]] shape, in the one
278
+ # place where the two answers are "may read" and "may not".
279
+ for a in existing.values():
280
+ if isinstance(a, dict) and str(a.get("channel") or "") == channel:
281
+ raise err(409, "channel_taken",
282
+ "that channel already has an agent — edit it instead of adding a second")
283
+ aid = secrets.token_urlsafe(9)
284
+ rec = _clean_agent(dict(body, id=aid, createdBy=session.uname,
285
+ createdAt=datetime.now(timezone.utc).isoformat(timespec="seconds")))
286
+ _put_agent(session, aid, lambda _prior: rec)
287
+ fresh = _agents(session.runtime).get(aid)
288
+ if not isinstance(fresh, dict):
289
+ # The store took the write and did not record it. A 200 here would tell an administrator
290
+ # the agent exists when it does not.
291
+ raise err(503, "store_unavailable", "the agent was NOT created")
292
+ return {"agent": {"id": aid, "channel": rec["channel"], "channelName": rec["channelName"],
293
+ "label": rec["label"], "active": rec["active"],
294
+ "writeApproval": rec["writeApproval"], "summary": "No access"}}
295
+
296
+
297
+ @router.get("/agents/{agent_id}")
298
+ def get_channel_agent(agent_id: str, session: Session = Depends(admin_gate)):
299
+ """The agent PLUS the permission envelope — the SAME shape `get_perms` answers with.
300
+
301
+ ⭐ ONE PAYLOAD SHAPE FOR TWO ROOMS. `permsModel.parsePermsPayload` parses this, and
302
+ `settings/ModulePermsList` renders it, because they are literally the same client code. A
303
+ second envelope here would mean a second parser and a second set of defaults within a wave.
304
+ """
305
+ import routes_admin
306
+ a = _agents(session.runtime).get(str(agent_id))
307
+ if not isinstance(a, dict):
308
+ raise err(404, "no_such_agent", "no agent with that id")
309
+ modules = routes_admin._perm_modules(session)
310
+ # ⭐⭐ W36-T22 / C2 — EVERY listed database, `ut_*` included. ⛔ THIS IS WHY THE FLAG'S
311
+ # DELETION IS NOT A ONE-FILE CHANGE: `m.get("enforced")` on a row that no longer carries the
312
+ # key is None, so this list would have been EMPTY and the channel perms editor would have
313
+ # governed ZERO databases while looking entirely correct. A Slack channel is a principal of
314
+ # the SAME wall (`verify_perm_scope` section H) and gets the same catalogue.
315
+ governed = [m["key"] for m in modules]
316
+ stored = a.get("perms") or {}
317
+ principal = agent_principal(a)
318
+ import core.perm_scope as perm_scope
319
+ perms_out = {}
320
+ for k in governed:
321
+ e = stored.get(k)
322
+ # ⛔ W36-T22 — `may_read`, the same evaluator the read door uses, for the reason spelled
323
+ # out at `routes_admin.get_perms`. ⚠ It answers DIFFERENTLY here and correctly so: a
324
+ # channel agent is not a person with a share, so a `ut_*` key it has not been granted
325
+ # defaults CLOSED — which is the fail-closed direction for a bot, and the same answer the
326
+ # table routes would give it.
327
+ perms_out[k] = e if isinstance(e, dict) else {
328
+ "access": bool(perm_scope.may_read(principal, k, st=session.runtime)),
329
+ "filter": None, "hiddenFields": []}
330
+ return {"id": str(agent_id), "channel": a.get("channel") or "",
331
+ "channelName": a.get("channelName") or "", "label": a.get("label") or "",
332
+ "active": a.get("active", True) is not False,
333
+ "writeApproval": a.get("writeApproval", True) is not False,
334
+ "perms": perms_out,
335
+ # ⛔ ALWAYS THE CURRENT VERSION, because `agent_principal` forces it: the editor must
336
+ # render "undeclared = denied", not the legacy-grant reading it would show at 0.
337
+ "perms_v": perm_scope.PERMS_VERSION,
338
+ # A bot is NEVER an admin — see `agent_principal`. Sent so the editor never paints
339
+ # the "Everything (admin)" state for a principal that cannot have it.
340
+ "is_admin": False,
341
+ "modules": modules,
342
+ "fields_by_module": {k: routes_admin._module_fields(k, session=session)
343
+ for k in governed}}
344
+
345
+
346
+ @router.put("/agents/{agent_id}/perms")
347
+ def put_channel_agent_perms(agent_id: str, body: dict = Body(default=None),
348
+ session: Session = Depends(admin_gate)):
349
+ """Replace this agent's permission block wholesale — the same validator a user's block gets."""
350
+ import routes_admin
351
+ body = body if isinstance(body, dict) else {}
352
+ if not isinstance(_agents(session.runtime).get(str(agent_id)), dict):
353
+ raise err(404, "no_such_agent", "no agent with that id")
354
+ if "perms" not in body:
355
+ raise err(400, "empty_patch", "no perms to save")
356
+ mods = routes_admin._perm_modules(session)
357
+ # ⭐ THE SAME `_clean_perms`, NOT A COPY OF IT. It refuses a filter this module cannot
358
+ # evaluate, refuses a hiddenFields key that names nothing, and refuses a BU condition the
359
+ # pushdown cannot read. Every one of those is exactly as true for a channel as for a person,
360
+ # and a second validator here is a second place for one of them to go missing.
361
+ # ⚠ CORRECTED W36-T22: this list used to end "refuses an unenforced `ut_*` key". That refusal
362
+ # is DELETED — W36-T21 armed the wall over every database, so there is no unenforced key left
363
+ # to refuse, and a comment naming a rule that no longer exists is how the next wave re-derives
364
+ # it ([[two-gates-can-assert-opposite-things]]).
365
+ cleaned = routes_admin._clean_perms(
366
+ body.get("perms"),
367
+ governed_keys={m["key"] for m in mods}, session=session) or {}
368
+
369
+ import core.perm_scope as perm_scope
370
+
371
+ def _mut(prior):
372
+ if not isinstance(prior, dict):
373
+ return None
374
+ nxt = dict(prior)
375
+ nxt["perms"] = cleaned
376
+ nxt["perms_v"] = perm_scope.PERMS_VERSION
377
+ return nxt
378
+
379
+ _put_agent(session, str(agent_id), _mut)
380
+ fresh = _agents(session.runtime).get(str(agent_id)) or {}
381
+ if (fresh.get("perms") or {}) != cleaned:
382
+ # The one direction this must never fail in: telling an administrator the wall is up when
383
+ # it is not.
384
+ raise err(503, "store_unavailable", "the permissions were not saved — the agent is UNCHANGED")
385
+ return {"id": str(agent_id), "perms": fresh.get("perms") or {},
386
+ "perms_v": int(fresh.get("perms_v") or 0)}
387
+
388
+
389
+ @router.patch("/agents/{agent_id}")
390
+ def patch_channel_agent(agent_id: str, body: dict = Body(default=None),
391
+ session: Session = Depends(admin_gate)):
392
+ body = body if isinstance(body, dict) else {}
393
+ if not isinstance(_agents(session.runtime).get(str(agent_id)), dict):
394
+ raise err(404, "no_such_agent", "no agent with that id")
395
+ #: ⛔ `perms` IS NOT PATCHABLE HERE. It has its own door with its own validator; accepting it
396
+ #: on the metadata route would be a second, unvalidated way to write the wall.
397
+ allowed = {k: v for k, v in body.items()
398
+ if k in ("label", "active", "writeApproval", "channelName")}
399
+ if not allowed:
400
+ raise err(400, "empty_patch", "nothing to change")
401
+
402
+ def _mut(prior):
403
+ return _clean_agent(allowed, prior=prior) if isinstance(prior, dict) else None
404
+
405
+ _put_agent(session, str(agent_id), _mut)
406
+ import routes_admin
407
+ a = _agents(session.runtime).get(str(agent_id)) or {}
408
+ return {"agent": {"id": str(agent_id), "channel": a.get("channel") or "",
409
+ "channelName": a.get("channelName") or "", "label": a.get("label") or "",
410
+ "active": a.get("active", True) is not False,
411
+ "writeApproval": a.get("writeApproval", True) is not False,
412
+ "summary": _summary(a, routes_admin._perm_modules(session))}}
413
+
414
+
415
+ @router.delete("/agents/{agent_id}")
416
+ def delete_channel_agent(agent_id: str, session: Session = Depends(admin_gate)):
417
+ if not isinstance(_agents(session.runtime).get(str(agent_id)), dict):
418
+ raise err(404, "no_such_agent", "no agent with that id")
419
+ _put_agent(session, str(agent_id), lambda _prior: None)
420
+ if isinstance(_agents(session.runtime).get(str(agent_id)), dict):
421
+ raise err(503, "store_unavailable", "the agent was NOT removed")
422
+ return {"ok": True}
423
+
424
+
425
+ # ── ⭐ W33-T67: THE WRITE-APPROVAL GATE ───────────────────────────────────────────────────────
426
+ def _is_empty(value):
427
+ """Is this value ABSENT, as opposed to falsy?
428
+
429
+ ⛔ COPIED EXACTLY FROM OpenTag's `agent/write_confirmation.py`, and the exactness is the point:
430
+ `0` and `False` are VALUES a human must see on the card. Setting a price to 0 or a flag to
431
+ False is precisely the kind of write somebody needs to approve, and a naive `if not value`
432
+ drops both rows silently — the approver then reads a card that does not describe the write
433
+ they are approving. The MIT licence gate for this borrowing is recorded in
434
+ `.claude/wiki/research/opentag-adoption.md`.
435
+ """
436
+ return value is None or (isinstance(value, str) and value.strip() == "")
437
+
438
+
439
+ def approval_card(action, values, agent=None):
440
+ """The card a human reads before a bot writes. `{action, rows, truncated, note}`.
441
+
442
+ ⛔ ROW-CAPPED, and the cap is a SAFETY property rather than a layout one: past a certain
443
+ length Slack collapses a message behind "Show more", and **an approver who cannot read the
444
+ card cannot approve it.** When rows are dropped the card SAYS how many — a truncation nobody
445
+ was told about is the violation, not the truncation (wave-30 R6's second sentence).
446
+ """
447
+ rows = [{"field": str(k), "value": v}
448
+ for k, v in (values or {}).items() if not _is_empty(v)]
449
+ rows.sort(key=lambda r: r["field"])
450
+ shown, dropped = rows[:APPROVAL_MAX_ROWS], max(0, len(rows) - APPROVAL_MAX_ROWS)
451
+ return {
452
+ "action": str(action or "")[:80],
453
+ "rows": shown,
454
+ "truncated": dropped,
455
+ "note": (f"{dropped} more field(s) not shown — open the record to review them before "
456
+ f"approving." if dropped else ""),
457
+ "agent": (agent or {}).get("label") or (agent or {}).get("channel") or "",
458
+ }
459
+
460
+
461
+ def intercept_write(agent, action, values, stash):
462
+ """THE INTERCEPTOR. Returns `("commit", None)` or `("await_approval", card)`.
463
+
464
+ ⭐ The one thing worth taking from OpenTag: a MUTATING tool call is halted, rendered as a card
465
+ naming the action and its values, and committed only on an explicit human accept. Everything
466
+ else in that repo — the transport, the runtime, the identity model — is AVOID (amendment A1).
467
+
468
+ ⛔ FAIL-CLOSED ON A MISSING FLAG: `is not False`, so a record written before `writeApproval`
469
+ existed requires approval. The unsafe direction here is not recoverable — revoking a bot does
470
+ not un-write what it wrote.
471
+ """
472
+ if (agent or {}).get("writeApproval", True) is not False:
473
+ card = approval_card(action, values, agent)
474
+ stash(card)
475
+ return "await_approval", card
476
+ return "commit", None
477
+
478
+
479
+ def _pending(rt):
480
+ try:
481
+ found = rt.get(PENDING_KEY) or {}
482
+ except Exception: # noqa: BLE001
483
+ return {}
484
+ return found if isinstance(found, dict) else {}
485
+
486
+
487
+ def stash_pending(rt, agent, action, values, now=None):
488
+ """Store one awaiting-approval mutation and return its token.
489
+
490
+ The token is what rides the card's button, so it is `secrets.token_urlsafe` and not the
491
+ action's id: a guessable value on an unauthenticated door is a way to approve somebody else's
492
+ write. Server-only bucket, per tenant, exactly as `routes_forms.TOKENS_KEY` is.
493
+ """
494
+ token = secrets.token_urlsafe(18)
495
+ row = {"at": float(now if now is not None else time.time()),
496
+ "channel": str((agent or {}).get("channel") or ""),
497
+ "action": str(action or "")[:80],
498
+ "values": values if isinstance(values, dict) else {}}
499
+ rt.update(PENDING_KEY, lambda cur: {**(cur or {}), token: row}, flush="sync")
500
+ return token
501
+
502
+
503
+ #: `{action kind: fn(rt, row) -> None}`. Registered by whatever can actually perform a mutation.
504
+ #:
505
+ #: ⛔⛔ EMPTY IN PRODUCTION TODAY, AND SAYING SO IS THE POINT. `commit_pending` below is the ONE
506
+ #: place an approved write may be performed, so it is the place a negative control can bite — but
507
+ #: a gate is only as honest as what it guards. Until something registers here, an approved card
508
+ #: performs NOTHING and `commit_pending` returns `no_committer` rather than pretending. That is a
509
+ #: declared absence, not a silent one [[flag-shipped-without-its-writer]].
510
+ _COMMITTERS = {}
511
+
512
+
513
+ def register_committer(kind, fn):
514
+ """Declare who may perform an approved mutation of `kind`. Idempotent by key."""
515
+ _COMMITTERS[str(kind)] = fn
516
+
517
+
518
+ def commit_pending(rt, token, approved, now=None):
519
+ """Resolve ONE approval card. Returns one of
520
+ `expired` · `unknown` · `rejected` · `no_committer` · `committed` · `failed`.
521
+
522
+ ⛔⛔ THIS IS THE WRITE GATE, AND IT IS A SEPARATE FUNCTION FROM THE ROUTE ON PURPOSE. T67's
523
+ `done-when` asks for an NC proving a REJECTED card writes nothing. A control aimed at the
524
+ route could not bite while the route had no write in it at all — it would stay green with the
525
+ guard deleted, because there would be nothing for the guard to be protecting anything from
526
+ ([[gate-can-report-green-on-nothing]], and a verifier caught exactly that on this ticket's
527
+ first draft). Putting the commit behind ONE named seam gives the control a real subject: the
528
+ gate registers a recorder, and `rejected` must leave it untouched while `approve` must call it
529
+ exactly once.
530
+
531
+ ⛔ THE TOKEN IS CONSUMED BEFORE EITHER BRANCH, and before any committer runs. A card that
532
+ survives its own click is a replayable write, and "approve twice" must never mean "write
533
+ twice". The delete is `flush="sync"` for the same reason.
534
+ """
535
+ now = float(now if now is not None else time.time())
536
+ row = _pending(rt).get(str(token))
537
+ if not isinstance(row, dict):
538
+ return "unknown"
539
+ # Consume FIRST — before the TTL verdict, before the approve/reject branch, before any write.
540
+ rt.update(PENDING_KEY,
541
+ lambda cur: {k: v for k, v in (cur or {}).items() if k != str(token)},
542
+ flush="sync")
543
+ if now - float(row.get("at") or 0) > APPROVAL_TTL_S:
544
+ return "expired"
545
+ if not approved:
546
+ # ⛔ NOTHING BELOW THIS LINE RUNS FOR A REJECTED CARD. The whole gate is this return.
547
+ return "rejected"
548
+ fn = _COMMITTERS.get(str(row.get("action") or ""))
549
+ if fn is None:
550
+ return "no_committer"
551
+ try:
552
+ fn(rt, row)
553
+ except Exception: # noqa: BLE001
554
+ # A committer that raises must not read as a commit. The card is already consumed, so the
555
+ # honest report is that it failed, and the human asks again.
556
+ return "failed"
557
+ return "committed"
558
+
559
+
560
+ # ── the UNAUTHENTICATED Slack door ───────────────────────────────────────────────────────────
561
+ #: Sliding window, keyed on the SLACK TEAM, never the client IP.
562
+ #:
563
+ #: ⛔ WHY NOT THE IP, WHICH IS WHAT `routes_forms` DOES. Every Slack event arrives from Slack's own
564
+ #: infrastructure, so an IP window is one shared bucket for every workspace on the platform: one
565
+ #: busy tenant would spend the allowance for all of them, and the symptom would be another
566
+ #: tenant's bot going quiet. The team id is the closest thing to the actual noisy party.
567
+ #: ⚠ It is attacker-CONTROLLED before the signature is checked, so the window is applied AFTER
568
+ #: verification, never before — an unsigned request is refused on cost grounds anyway (a signature
569
+ #: check is a single HMAC).
570
+ RATE_WINDOW_S, RATE_PER_WINDOW = 60, 60
571
+ _HITS: dict = {}
572
+
573
+
574
+ def _rate_ok(key, now):
575
+ seen = [t for t in _HITS.get(key, ()) if now - t < RATE_WINDOW_S]
576
+ seen.append(now)
577
+ _HITS[key] = seen
578
+ if len(_HITS) > 4096:
579
+ for k in [k for k, v in _HITS.items() if not v or now - v[-1] > RATE_WINDOW_S]:
580
+ _HITS.pop(k, None)
581
+ return len(seen) <= RATE_PER_WINDOW
582
+
583
+
584
+ async def _raw_body(request):
585
+ """The body as RAW BYTES, capped.
586
+
587
+ ⛔⛔ RAW, NOT PARSED, AND THAT IS NOT A STYLE CHOICE. Slack signs the literal string
588
+ `v0:{timestamp}:{body}` byte for byte. `routes_forms._bounded_body` — the function this is
589
+ otherwise copied from — returns only the parsed dict and DISCARDS the bytes, so a verbatim
590
+ copy of it could not verify a Slack signature at all: re-serialising the dict changes
591
+ whitespace and key order and the HMAC no longer matches. The bug would look like "Slack
592
+ signatures are always invalid", which is indistinguishable from a wrong secret.
593
+
594
+ ⚠ And it streams rather than calling `request.body()`: FastAPI would otherwise have buffered
595
+ the whole body before the handler's first line, so a content-length check inside the handler
596
+ caps nothing (`routes_forms`' own scar, recorded in its docstring).
597
+ """
598
+ size, chunks = 0, []
599
+ async for chunk in request.stream():
600
+ size += len(chunk)
601
+ if size > MAX_BODY_BYTES:
602
+ raise err(413, "body_too_large", "that request is too large")
603
+ chunks.append(chunk)
604
+ return b"".join(chunks)
605
+
606
+
607
+ def _refuse():
608
+ """ONE refusal for every reason. No oracle.
609
+
610
+ A wrong signature, a stale timestamp, an unknown team, an unconfigured tenant and a channel
611
+ with no agent all answer identically — otherwise the door tells an unauthenticated caller
612
+ which tenants exist and which channels are configured, which is the question it was built to
613
+ not answer. `routes_forms._refuse` takes the same posture for the same reason.
614
+ """
615
+ return err(403, "bad_request", "that request could not be verified")
616
+
617
+
618
+ def _verify(secret, timestamp, raw, signature, now):
619
+ """Slack's v0 signature, with the replay window checked FIRST."""
620
+ try:
621
+ ts = int(str(timestamp or "0"))
622
+ except ValueError:
623
+ return False
624
+ if abs(now - ts) > SLACK_MAX_SKEW_S:
625
+ return False
626
+ base = b"v0:" + str(ts).encode("ascii") + b":" + (raw or b"")
627
+ import hashlib
628
+ expected = "v0=" + hmac.new(str(secret).encode("utf-8"), base, hashlib.sha256).hexdigest()
629
+ return hmac.compare_digest(expected, str(signature or ""))
630
+
631
+
632
+ def _resolve_tenant(raw, timestamp, signature, now):
633
+ """`(slug, runtime, creds)` for the tenant whose signing secret verifies this request.
634
+
635
+ ⛔ THE TENANT IS DECIDED BY THE SIGNATURE, NEVER BY THE PAYLOAD. A body-supplied `team_id`
636
+ would let an unauthenticated caller name the tenant it wants to be — the exact widening
637
+ `deps._user_for`'s tenant-equality check closes on the authenticated side. So every configured
638
+ tenant's secret is tried and the one that VERIFIES names the tenant.
639
+
640
+ ⚠ NO EARLY `break` ON A FAILED CANDIDATE and no per-tenant error: the loop's cost must not
641
+ depend on which tenant matched. `routes_forms._resolve` walks tenants the same way, for the
642
+ same reason — a public door has no session to ask.
643
+ """
644
+ from harness import runtime as _rt
645
+ hit = None
646
+ try:
647
+ slugs = list(_rt.known_tenants())
648
+ except Exception: # noqa: BLE001
649
+ return None, None, None
650
+ for slug in slugs:
651
+ try:
652
+ rt = _rt.get_runtime(slug)
653
+ except Exception: # noqa: BLE001
654
+ continue
655
+ creds = slack_creds(rt)
656
+ if not creds:
657
+ continue
658
+ if _verify(creds.get("signing_secret"), timestamp, raw, signature, now) and hit is None:
659
+ hit = (slug, rt, creds)
660
+ return hit if hit else (None, None, None)
661
+
662
+
663
+ @router.post("/slack/events")
664
+ async def slack_events(request: Request):
665
+ """Slack's Events API. UNAUTHENTICATED BY CONSTRUCTION — no `Depends(require_session)`.
666
+
667
+ ⭐ Built like `routes_forms.py`'s public door and not like OpenTag's: one process, no vendor,
668
+ no second runtime, no persistent outbound socket, and this tenant's Slack secret never leaves
669
+ this deployment (amendment A1).
670
+ """
671
+ now = time.time()
672
+ raw = await _raw_body(request)
673
+ ts = request.headers.get("x-slack-request-timestamp")
674
+ sig = request.headers.get("x-slack-signature")
675
+ slug, rt, _creds = _resolve_tenant(raw, ts, sig, now)
676
+ if not rt:
677
+ raise _refuse()
678
+
679
+ try:
680
+ payload = json.loads(raw or b"{}")
681
+ except ValueError:
682
+ raise _refuse()
683
+ if not isinstance(payload, dict):
684
+ raise _refuse()
685
+
686
+ # Slack's one-time endpoint handshake. Answered ONLY after the signature verified — an
687
+ # unsigned challenge echo would confirm the endpoint exists to anyone who probes it.
688
+ if payload.get("type") == "url_verification":
689
+ return {"challenge": str(payload.get("challenge") or "")[:512]}
690
+
691
+ team = str(payload.get("team_id") or slug)
692
+ if not _rate_ok(f"{slug}:{team}", now):
693
+ raise err(429, "too_many_requests", "too many requests — wait a moment and try again")
694
+
695
+ event = payload.get("event") if isinstance(payload.get("event"), dict) else {}
696
+ channel = str(event.get("channel") or "")
697
+ agent = None
698
+ for a in _agents(rt).values():
699
+ if isinstance(a, dict) and str(a.get("channel") or "") == channel:
700
+ agent = a
701
+ break
702
+ # ⛔ NO AGENT, OR AN INACTIVE ONE, IS SILENCE — a 200 with no action. Not a 403: Slack retries
703
+ # a non-2xx up to three times, so refusing here would turn "this channel is not configured"
704
+ # into three refusals per message, and the retry storm would be the only visible symptom.
705
+ # ⚠ And it must not say WHICH — a distinguishable answer tells an unauthenticated caller which
706
+ # channels this workspace has configured.
707
+ if not isinstance(agent, dict) or agent.get("active", True) is False:
708
+ return {"ok": True}
709
+
710
+ # ⛔⛔ THE ANSWERING HALF IS NOT BUILT YET, AND SAYING SO IS THE POINT. What ships here is the
711
+ # DOOR and the WALL: a verified request, resolved to a tenant by signature, matched to a
712
+ # channel agent whose `perms` block is a `perm_scope` principal (`agent_principal`). Composing
713
+ # a reply means an LLM call on the cheap-first ladder plus the read path, and a half-built
714
+ # answerer that returns something plausible is worse than one that returns nothing.
715
+ # Booked rather than faked [[flag-shipped-without-its-writer]].
716
+ return {"ok": True}
717
+
718
+
719
+ @router.post("/slack/interact")
720
+ async def slack_interact(request: Request):
721
+ """The approval card's button (`W33-T67`). UNAUTHENTICATED, signature-verified, same shape."""
722
+ now = time.time()
723
+ raw = await _raw_body(request)
724
+ slug, rt, _creds = _resolve_tenant(raw, request.headers.get("x-slack-request-timestamp"),
725
+ request.headers.get("x-slack-signature"), now)
726
+ if not rt:
727
+ raise _refuse()
728
+ # Slack posts interactions as `application/x-www-form-urlencoded` with a `payload=` field.
729
+ from urllib.parse import parse_qs
730
+ try:
731
+ form = parse_qs(raw.decode("utf-8"))
732
+ payload = json.loads((form.get("payload") or ["{}"])[0])
733
+ except Exception: # noqa: BLE001
734
+ raise _refuse()
735
+ if not isinstance(payload, dict):
736
+ raise _refuse()
737
+ if not _rate_ok(f"{slug}:interact", now):
738
+ raise err(429, "too_many_requests", "too many requests — wait a moment and try again")
739
+
740
+ actions = payload.get("actions") if isinstance(payload.get("actions"), list) else []
741
+ choice = (actions[0] if actions and isinstance(actions[0], dict) else {})
742
+ token = str(choice.get("value") or "")
743
+ # ⭐ ANYTHING THAT IS NOT LITERALLY "approve" IS A REJECT. Fail-closed on a malformed,
744
+ # truncated or unknown action id: the direction that can be wrong here without anyone
745
+ # noticing is the one that writes.
746
+ approved = str(choice.get("action_id") or "") == "approve"
747
+ # ⛔ THE ROUTE DOES NOT DECIDE — `commit_pending` does, and it is the ONE seam a negative
748
+ # control can aim at. Consumption, the TTL, the reject branch and the committer lookup all
749
+ # live there; this handler only turns its verdict into a sentence.
750
+ verdict = commit_pending(rt, token, approved, now=now)
751
+ return {"text": {
752
+ # An unknown, already-used or expired token is not an error: a person clicking a stale
753
+ # card should be told it is stale, not shown a failure.
754
+ "unknown": "That approval is no longer available. Ask the bot again.",
755
+ "expired": "That approval expired before it was answered. Ask the bot again.",
756
+ "rejected": "Rejected. Nothing was changed.",
757
+ # ⚠ HONEST, not reassuring: the answering half that would register a committer is booked,
758
+ # so an approved card today performs nothing and says exactly that rather than "Approved."
759
+ "no_committer": "Approved — but this workspace has nothing configured to carry it out yet.",
760
+ "failed": "That change could not be completed. Nothing was saved; ask the bot again.",
761
+ "committed": "Approved.",
762
+ }.get(verdict, "That approval is no longer available. Ask the bot again.")}
763
+
764
+
765
+ @router.get("/slack/health")
766
+ def slack_health(session: Session = Depends(require_session)):
767
+ """Is Slack configured for THIS tenant? Session-gated, and it answers about one tenant only —
768
+ enumerating the others would answer a question about our customer list."""
769
+ creds = slack_creds(session.runtime)
770
+ return {"configured": creds is not None,
771
+ "agents": len(_agents(session.runtime)),
772
+ "hasBotToken": bool((creds or {}).get("bot_token"))}
api/routes_statements.py CHANGED
@@ -1,298 +1,298 @@
1
- """routes_statements.py — the statement-of-account sender, ported off Streamlit (EXIT-6).
2
-
3
- ⛔ THIS IS THE ONE SANCTIONED ODOO WRITER IN THE ENTIRE SYSTEM. Everything else in AIOS is
4
- read-only on Odoo by hard block. `modules/collections_send.py` owns a narrow client that whitelists
5
- exactly `mail.mail create` (queueing an outbound email) and nothing else; these routes call it and
6
- add no write of their own.
7
-
8
- WHY IT EXISTS SEPARATELY FROM THE COLLECTIONS PAGE. Owner ruling wave-17 item 15 retired the
9
- Collections *dashboard* — the worklist is the shared "Collections" view on the Customer grid, from
10
- the same reconciled blocks. What that ruling explicitly left "untouched" is this send workflow, so
11
- when `app.py` is deleted it is the ONLY live Streamlit-only feature, and it moves here rather than
12
- dying with the host. Ported faithfully: same tiers, same filters, same template placeholders, same
13
- preview, same test-send, same two-step confirm.
14
-
15
- THE THREE GUARDRAILS, and where each is enforced:
16
- 1. SAFE_MODE (default ON) — in the DATA LAYER (`collections_send.queue_statement`), so no route,
17
- payload or UI can bypass it. These routes only REPORT it; they never re-implement the check.
18
- 2. Admin only — `admin_gate` (role, fail-closed), mirroring the Streamlit `if not is_admin()`.
19
- 3. ⭐ TENANT — NEW HERE, and it did not exist in Streamlit because it could not. `cs_mod.Odoo()`
20
- reads Odoo credentials from the ENVIRONMENT, which after the keychain cutover belongs to
21
- TENANT #0 ALONE. On the multi-tenant API an unguarded route would let a nurilab or gtmlab
22
- admin queue mail from Royal Imports' Odoo, as Royal Imports. `_royal_only` closes that; the
23
- single-tenant Streamlit host never had the exposure, so this is a port that must ADD a wall
24
- rather than copy one.
25
-
26
- NOTHING SENDS ON A GET. The send route requires an explicit customer list in the body; there is no
27
- "send all" parameter, deliberately — the confirm step is a product requirement, not a formality.
28
- """
29
- from fastapi import APIRouter, Body, Depends
30
-
31
- from deps import Session, err
32
- from routes_admin import admin_gate
33
-
34
- # W35-T35 (C8): the tenant predicate, held once. See `_royal_only` for why the import runs this
35
- # way round. `main.py` already imports both modules, so this adds no load.
36
- import automation_engine as engine
37
-
38
- router = APIRouter(prefix="/api/v1")
39
-
40
- #: Cache the Odoo follow-up pull briefly. The Streamlit page used `@st.cache_data(ttl=1800)`; the
41
- #: list moves slowly (it is a dunning worklist, not a live feed) and the pull is a multi-model read.
42
- _TTL = 1800
43
- _cache = {"at": 0.0, "rows": None}
44
-
45
-
46
- def _cs():
47
- import modules.collections_send as cs
48
- return cs
49
-
50
-
51
- def _royal_only(session: Session) -> Session:
52
- """⛔ See the module docstring, guardrail 3. The send client is env-credentialed, so it is
53
- tenant #0's and only tenant #0's. Refuse for anyone else rather than send as the wrong company.
54
-
55
- Keyed on the runtime, never on a request field: a tenant is a property of the SESSION.
56
-
57
- ⭐⭐ WAVE 35 · T35 / CONTRACT C8 — THE TEST ITSELF NOW LIVES IN ONE PLACE. Statements became an
58
- agent step this wave, so the automation engine has to answer the same question at two more
59
- doors (which tenants see the action, which tenants may store it). C8's words are "the predicate
60
- is imported, never re-expressed", and this is the direction that import can run: the engine
61
- imports no route module and no FastAPI, so reaching `_royal_only` FROM it would drag `deps` and
62
- `routes_admin` into a route-free module. The engine holds the boolean; this holds the refusal.
63
- ⚠ WHAT STAYS HERE IS THE HTTP SHAPE, and that is deliberate — a 404 rather than a 403, so the
64
- surface is invisible rather than forbidden. The engine must not know about status codes."""
65
- if not engine.is_statement_tenant(session.runtime):
66
- raise err(404, "not_found", "statements are not configured for this workspace")
67
- return session
68
-
69
-
70
- def _gate(session: Session = Depends(admin_gate)) -> Session:
71
- return _royal_only(session)
72
-
73
-
74
- def _rows(force=False):
75
- import time
76
- cs = _cs()
77
- if force or _cache["rows"] is None or (time.time() - _cache["at"]) > _TTL:
78
- _cache["rows"] = cs.load_collection_list(cs.Odoo())
79
- _cache["at"] = time.time()
80
- return _cache["rows"], time.strftime("%Y-%m-%d %H:%M", time.localtime(_cache["at"]))
81
-
82
-
83
- def _public(row):
84
- """Strip the internals the Streamlit grid also hid (`_`-prefixed + partner_id is kept, because
85
- the client needs a stable row identity that is not the display name)."""
86
- return {k: v for k, v in row.items() if not str(k).startswith("_")}
87
-
88
-
89
- @router.get("/admin/statements")
90
- def statements(refresh: int = 0, session: Session = Depends(_gate)):
91
- """The worklist + everything the sender UI needs to render itself honestly."""
92
- cs = _cs()
93
- try:
94
- rows, loaded_at = _rows(force=bool(refresh))
95
- except Exception as e:
96
- raise err(502, "odoo_unavailable", f"could not load the collection list: {str(e)[:200]}")
97
- return {
98
- "rows": [_public(r) for r in rows],
99
- "loadedAt": loaded_at,
100
- # The guardrail is reported, never decided, here — the data layer owns it.
101
- "safeMode": bool(cs.SAFE_MODE),
102
- "safeRecipients": sorted(cs.SAFE_RECIPIENTS),
103
- "sender": {"name": cs.SENDER_NAME, "email": cs.SENDER_EMAIL,
104
- "replyTo": cs.REPLY_TO, "company": cs.COMPANY},
105
- "templates": {"subject": cs.DEFAULT_SUBJECT, "intro": cs.DEFAULT_INTRO,
106
- "footer": cs.DEFAULT_FOOTER},
107
- # W35-T35: ONE literal, read from the engine, so the sender's worklist and a statements
108
- # agent's tier filter cannot come to mean different things.
109
- "tiers": list(engine.STATEMENT_TIERS),
110
- }
111
-
112
-
113
- def _find(rows, customer):
114
- return next((r for r in rows if r.get("Customer") == customer), None)
115
-
116
-
117
- @router.post("/admin/statements/preview")
118
- def preview(body: dict = Body(default=None), session: Session = Depends(_gate)):
119
- """Render ONE customer's statement exactly as the send path would."""
120
- cs = _cs()
121
- body = body or {}
122
- rows, _ = _rows()
123
- row = _find(rows, body.get("customer"))
124
- if row is None:
125
- raise err(404, "not_found", "no such customer on the collection list")
126
- t = body.get("templates") or {}
127
- import datetime as dt
128
- month = dt.date.today().strftime("%B %Y")
129
- subject = (t.get("subject") or cs.DEFAULT_SUBJECT)
130
- try:
131
- subject = subject.format(customer=row["Customer"], company=cs.COMPANY, month=month)
132
- except Exception: # noqa: BLE001
133
- # An unknown placeholder is the user's typo, not a 500. Show the template verbatim so they
134
- # can see what they typed rather than getting an opaque error.
135
- # ⛔ W36-T42: THIS CAUGHT THREE OF THE WAYS `str.format` FAILS AND THERE ARE MORE.
136
- # `{customer` is a ValueError, `{customer.x}` an AttributeError, `{customer:%Y}` a
137
- # TypeError, and every one of them is the same user mistake this branch was written for.
138
- # Naming a subset of the exception types turns a typo in the OTHER half into a 500 on the
139
- # one screen somebody opens to avoid mailing 200 people the wrong thing.
140
- pass
141
- return {
142
- "html": cs.render_statement_html(row, t.get("intro") or cs.DEFAULT_INTRO,
143
- t.get("footer") or cs.DEFAULT_FOOTER),
144
- "to": row.get("Email") or "",
145
- "subject": subject,
146
- }
147
-
148
-
149
- @router.post("/admin/statements/send")
150
- def send(body: dict = Body(default=None), session: Session = Depends(_gate)):
151
- """Queue statements. Returns per-customer outcomes — NEVER a bare count.
152
-
153
- `overrideTo` is the test-send path: one customer, one address. SAFE_MODE still applies (the
154
- data layer refuses an address outside the allow-list), which is why the route does not check it.
155
- """
156
- cs = _cs()
157
- body = body or {}
158
- names = [str(n) for n in (body.get("customers") or []) if str(n).strip()]
159
- if not names:
160
- raise err(400, "bad_request", "name at least one customer")
161
- # ⛔⛔ W36-T42 / D-299 — A DECLARED TEST SEND WITH NO ADDRESS REFUSES. IT DOES NOT BECOME A
162
- # REAL ONE. This read `(body.get("overrideTo") or "").strip() or None`, so a caller who sent
163
- # `overrideTo: ""` (the field left blank, the state not yet typed into, a trimmed-away space)
164
- # got a REAL statement mailed to the debtor's own address, reported back as `test: false`. On
165
- # the one route in this product that is allowed to write to Odoo, the difference between a
166
- # rehearsal and mailing a live customer was one empty string.
167
- #
168
- # ⭐ THE DISCRIMINATOR IS THE CALLER'S DECLARATION, NOT THE VALUE. A body with no `overrideTo`
169
- # key and no `test` flag is the PRODUCTION send, and refusing that would delete the feature
170
- # D-299 exists to preserve. What can be refused is a caller who SAID this is a test: the key
171
- # being present, or `test: true`, is that statement, and an empty address beside it is the
172
- # mistake. So both spellings of "absent" are covered: the key present and blank, and `test`
173
- # asserted with no key at all.
174
- #
175
- # ⚠ THIS IS THE SECOND OF THREE WALLS AND THE ONLY ONE A PAYLOAD CANNOT ROUTE AROUND.
176
- # `automationApi.testSendStatement` refuses an empty address before the request is built, and
177
- # SAFE_MODE refuses an address outside the allow-list inside `queue_statement`. The client
178
- # wall is bypassable by construction (anything can POST); this one is not.
179
- declared_test = ("overrideTo" in body) or bool(body.get("test"))
180
- override = str(body.get("overrideTo") or "").strip()
181
- if declared_test and not override:
182
- raise err(400, "no_override_address",
183
- "a test send needs the address to send the test to. Without one this would "
184
- "mail the customer's own address, which is the opposite of a test")
185
- override = override or None
186
- if override and len(names) != 1:
187
- raise err(400, "bad_request", "a test send takes exactly one customer")
188
- t = body.get("templates") or {}
189
- rows, _ = _rows()
190
-
191
- sent, failed, skipped = [], [], []
192
- for name in names:
193
- row = _find(rows, name)
194
- if row is None:
195
- failed.append({"customer": name, "error": "not on the current collection list"})
196
- continue
197
- if not override and not row.get("Email"):
198
- # The Streamlit page warned and skipped these. Reporting them SEPARATELY from failures
199
- # keeps "we could not" distinct from "there was nowhere to send".
200
- skipped.append({"customer": name, "reason": "no email address on the customer record"})
201
- continue
202
- try:
203
- mid = cs.queue_statement(cs.Odoo(), row, t.get("subject") or cs.DEFAULT_SUBJECT,
204
- t.get("intro") or cs.DEFAULT_INTRO,
205
- t.get("footer") or cs.DEFAULT_FOOTER, override_to=override)
206
- sent.append({"customer": name, "to": override or row.get("Email"), "mailId": mid})
207
- except Exception as e:
208
- # SafeModeBlocked lands here too, and that is correct: to the caller a guardrail refusal
209
- # and an Odoo error are both "this one did not go", each with its own honest message.
210
- failed.append({"customer": name, "error": str(e)[:200]})
211
- return {"sent": sent, "failed": failed, "skipped": skipped,
212
- "safeMode": bool(cs.SAFE_MODE), "test": bool(override)}
213
-
214
-
215
- # ---------------------------------------------------------------------------------------------
216
- # WAVE 35 · T36 / CONTRACT C8 / OWNER RULING R10 — THE AGENT'S PARKED BATCH.
217
- #
218
- # ⛔⛔ THE AGENT ASSEMBLES; A PERSON CLICKS SEND. `automation_engine.run_statements` renders a batch
219
- # and parks it on the automation definition, and it imports no mail path at all. THIS is the only
220
- # door that releases one, and it is deliberately built out of the parts already here:
221
- # · `_gate` — `admin_gate` THEN `_royal_only`, byte-for-byte the dependency the other three
222
- # endpoints use. Not a copy of the checks: the same object.
223
- # · `cs.queue_statement` — the same call `send()` above makes, so SAFE_MODE's allow-list refusal
224
- # happens in the DATA LAYER, where no route, payload or UI can reach around it.
225
- # ⇒ Nothing about the guardrails is re-expressed here, which is what makes "unchanged" checkable.
226
- #
227
- # ⚠ WHY THE BATCH IS RE-RENDERED FROM THE LIVE WORKLIST RATHER THAN SENT AS STORED. The parked
228
- # `html` is what a person APPROVED and is what the review screen shows; but a balance can move
229
- # between the parking and the click, and mailing a figure we know to be stale is worse than mailing
230
- # a fresh one. So the parked batch decides WHO and WITH WHAT WORDS, and the live row decides the
231
- # NUMBERS — the same split `preview` already makes. A customer who has left the worklist entirely
232
- # (they paid) is reported as skipped rather than invoiced.
233
-
234
- @router.get("/admin/statements/agent/{auto_id}")
235
- def agent_batch(auto_id: str, session: Session = Depends(_gate)):
236
- """What is parked and waiting for a click, for ONE agent. `null` when nothing is."""
237
- defn = (engine.all_definitions(session.runtime) or {}).get(str(auto_id)) or {}
238
- pending = defn.get("pendingStatements")
239
- if not isinstance(pending, dict):
240
- return {"pending": None}
241
- # ⚠ The rendered `html` is NOT returned in the list — a 200-statement batch of rendered mail is
242
- # megabytes, and the review screen needs the count and the names to decide. `preview` already
243
- # renders ONE on demand.
244
- return {"pending": {
245
- "ts": pending.get("ts"), "count": int(pending.get("count") or 0),
246
- "notes": list(pending.get("notes") or []),
247
- "items": [{k: v for k, v in it.items() if k != "html"}
248
- for it in (pending.get("items") or [])],
249
- "safeMode": bool(_cs().SAFE_MODE)}}
250
-
251
-
252
- @router.post("/admin/statements/agent/{auto_id}/send")
253
- def agent_send(auto_id: str, body: dict = Body(default=None),
254
- session: Session = Depends(_gate)):
255
- """Release a parked batch. THE CLICK R10 REQUIRES — nothing else in the system calls this.
256
-
257
- ⛔ IT REFUSES AN EMPTY OR MISSING BATCH rather than answering 200 with nothing sent: a send
258
- that reports success and mails nobody is the `view_upsert` failure mode (200 OK, zero writes)
259
- arriving on the one route in this system that talks to a mail server.
260
- """
261
- cs = _cs()
262
- defn = (engine.all_definitions(session.runtime) or {}).get(str(auto_id)) or {}
263
- pending = defn.get("pendingStatements")
264
- items = list((pending or {}).get("items") or []) if isinstance(pending, dict) else []
265
- if not items:
266
- raise err(404, "no_batch", "there is nothing parked for this agent to send")
267
- # ⚠ An explicit subset is allowed (a person unticking a customer on the review screen) but it
268
- # may only NARROW the parked batch. A name that was never parked cannot be introduced by the
269
- # payload, or the review stops being what authorised the send.
270
- want = {str(n) for n in (body or {}).get("customers") or []}
271
- if want:
272
- items = [it for it in items if str(it.get("customer")) in want]
273
- if not items:
274
- raise err(400, "bad_request", "none of those customers are in the parked batch")
275
- rows, _ = _rows()
276
- sent, failed, skipped = [], [], []
277
- for it in items:
278
- name = str(it.get("customer") or "")
279
- row = _find(rows, name)
280
- if row is None:
281
- skipped.append({"customer": name,
282
- "reason": "no longer on the collection list, so nothing is owed"})
283
- continue
284
- try:
285
- # ⛔ THE SAME DATA-LAYER CALL `send()` MAKES. SafeModeBlocked is raised INSIDE
286
- # `queue_statement`, so the guardrail cannot be argued with from here.
287
- mid = cs.queue_statement(cs.Odoo(), row,
288
- str(it.get("subject") or "") or cs.DEFAULT_SUBJECT,
289
- cs.DEFAULT_INTRO, cs.DEFAULT_FOOTER)
290
- sent.append({"customer": name, "to": row.get("Email"), "mailId": mid})
291
- except Exception as e: # noqa: BLE001
292
- failed.append({"customer": name, "error": str(e)[:200]})
293
- # ⚠ CLEARED ONLY WHEN NOTHING IS LEFT TO RETRY. A batch dropped while some of it failed would
294
- # lose the list of who still needs a statement, and nobody would know to look.
295
- if sent and not failed:
296
- engine.clear_statements(session.runtime, auto_id)
297
- return {"sent": sent, "failed": failed, "skipped": skipped,
298
- "safeMode": bool(cs.SAFE_MODE), "cleared": bool(sent and not failed)}
 
1
+ """routes_statements.py — the statement-of-account sender, ported off Streamlit (EXIT-6).
2
+
3
+ ⛔ THIS IS THE ONE SANCTIONED ODOO WRITER IN THE ENTIRE SYSTEM. Everything else in AIOS is
4
+ read-only on Odoo by hard block. `modules/collections_send.py` owns a narrow client that whitelists
5
+ exactly `mail.mail create` (queueing an outbound email) and nothing else; these routes call it and
6
+ add no write of their own.
7
+
8
+ WHY IT EXISTS SEPARATELY FROM THE COLLECTIONS PAGE. Owner ruling wave-17 item 15 retired the
9
+ Collections *dashboard* — the worklist is the shared "Collections" view on the Customer grid, from
10
+ the same reconciled blocks. What that ruling explicitly left "untouched" is this send workflow, so
11
+ when `app.py` is deleted it is the ONLY live Streamlit-only feature, and it moves here rather than
12
+ dying with the host. Ported faithfully: same tiers, same filters, same template placeholders, same
13
+ preview, same test-send, same two-step confirm.
14
+
15
+ THE THREE GUARDRAILS, and where each is enforced:
16
+ 1. SAFE_MODE (default ON) — in the DATA LAYER (`collections_send.queue_statement`), so no route,
17
+ payload or UI can bypass it. These routes only REPORT it; they never re-implement the check.
18
+ 2. Admin only — `admin_gate` (role, fail-closed), mirroring the Streamlit `if not is_admin()`.
19
+ 3. ⭐ TENANT — NEW HERE, and it did not exist in Streamlit because it could not. `cs_mod.Odoo()`
20
+ reads Odoo credentials from the ENVIRONMENT, which after the keychain cutover belongs to
21
+ TENANT #0 ALONE. On the multi-tenant API an unguarded route would let a nurilab or gtmlab
22
+ admin queue mail from Royal Imports' Odoo, as Royal Imports. `_royal_only` closes that; the
23
+ single-tenant Streamlit host never had the exposure, so this is a port that must ADD a wall
24
+ rather than copy one.
25
+
26
+ NOTHING SENDS ON A GET. The send route requires an explicit customer list in the body; there is no
27
+ "send all" parameter, deliberately — the confirm step is a product requirement, not a formality.
28
+ """
29
+ from fastapi import APIRouter, Body, Depends
30
+
31
+ from deps import Session, err
32
+ from routes_admin import admin_gate
33
+
34
+ # W35-T35 (C8): the tenant predicate, held once. See `_royal_only` for why the import runs this
35
+ # way round. `main.py` already imports both modules, so this adds no load.
36
+ import automation_engine as engine
37
+
38
+ router = APIRouter(prefix="/api/v1")
39
+
40
+ #: Cache the Odoo follow-up pull briefly. The Streamlit page used `@st.cache_data(ttl=1800)`; the
41
+ #: list moves slowly (it is a dunning worklist, not a live feed) and the pull is a multi-model read.
42
+ _TTL = 1800
43
+ _cache = {"at": 0.0, "rows": None}
44
+
45
+
46
+ def _cs():
47
+ import modules.collections_send as cs
48
+ return cs
49
+
50
+
51
+ def _royal_only(session: Session) -> Session:
52
+ """⛔ See the module docstring, guardrail 3. The send client is env-credentialed, so it is
53
+ tenant #0's and only tenant #0's. Refuse for anyone else rather than send as the wrong company.
54
+
55
+ Keyed on the runtime, never on a request field: a tenant is a property of the SESSION.
56
+
57
+ ⭐⭐ WAVE 35 · T35 / CONTRACT C8 — THE TEST ITSELF NOW LIVES IN ONE PLACE. Statements became an
58
+ agent step this wave, so the automation engine has to answer the same question at two more
59
+ doors (which tenants see the action, which tenants may store it). C8's words are "the predicate
60
+ is imported, never re-expressed", and this is the direction that import can run: the engine
61
+ imports no route module and no FastAPI, so reaching `_royal_only` FROM it would drag `deps` and
62
+ `routes_admin` into a route-free module. The engine holds the boolean; this holds the refusal.
63
+ ⚠ WHAT STAYS HERE IS THE HTTP SHAPE, and that is deliberate — a 404 rather than a 403, so the
64
+ surface is invisible rather than forbidden. The engine must not know about status codes."""
65
+ if not engine.is_statement_tenant(session.runtime):
66
+ raise err(404, "not_found", "statements are not configured for this workspace")
67
+ return session
68
+
69
+
70
+ def _gate(session: Session = Depends(admin_gate)) -> Session:
71
+ return _royal_only(session)
72
+
73
+
74
+ def _rows(force=False):
75
+ import time
76
+ cs = _cs()
77
+ if force or _cache["rows"] is None or (time.time() - _cache["at"]) > _TTL:
78
+ _cache["rows"] = cs.load_collection_list(cs.Odoo())
79
+ _cache["at"] = time.time()
80
+ return _cache["rows"], time.strftime("%Y-%m-%d %H:%M", time.localtime(_cache["at"]))
81
+
82
+
83
+ def _public(row):
84
+ """Strip the internals the Streamlit grid also hid (`_`-prefixed + partner_id is kept, because
85
+ the client needs a stable row identity that is not the display name)."""
86
+ return {k: v for k, v in row.items() if not str(k).startswith("_")}
87
+
88
+
89
+ @router.get("/admin/statements")
90
+ def statements(refresh: int = 0, session: Session = Depends(_gate)):
91
+ """The worklist + everything the sender UI needs to render itself honestly."""
92
+ cs = _cs()
93
+ try:
94
+ rows, loaded_at = _rows(force=bool(refresh))
95
+ except Exception as e:
96
+ raise err(502, "odoo_unavailable", f"could not load the collection list: {str(e)[:200]}")
97
+ return {
98
+ "rows": [_public(r) for r in rows],
99
+ "loadedAt": loaded_at,
100
+ # The guardrail is reported, never decided, here — the data layer owns it.
101
+ "safeMode": bool(cs.SAFE_MODE),
102
+ "safeRecipients": sorted(cs.SAFE_RECIPIENTS),
103
+ "sender": {"name": cs.SENDER_NAME, "email": cs.SENDER_EMAIL,
104
+ "replyTo": cs.REPLY_TO, "company": cs.COMPANY},
105
+ "templates": {"subject": cs.DEFAULT_SUBJECT, "intro": cs.DEFAULT_INTRO,
106
+ "footer": cs.DEFAULT_FOOTER},
107
+ # W35-T35: ONE literal, read from the engine, so the sender's worklist and a statements
108
+ # agent's tier filter cannot come to mean different things.
109
+ "tiers": list(engine.STATEMENT_TIERS),
110
+ }
111
+
112
+
113
+ def _find(rows, customer):
114
+ return next((r for r in rows if r.get("Customer") == customer), None)
115
+
116
+
117
+ @router.post("/admin/statements/preview")
118
+ def preview(body: dict = Body(default=None), session: Session = Depends(_gate)):
119
+ """Render ONE customer's statement exactly as the send path would."""
120
+ cs = _cs()
121
+ body = body or {}
122
+ rows, _ = _rows()
123
+ row = _find(rows, body.get("customer"))
124
+ if row is None:
125
+ raise err(404, "not_found", "no such customer on the collection list")
126
+ t = body.get("templates") or {}
127
+ import datetime as dt
128
+ month = dt.date.today().strftime("%B %Y")
129
+ subject = (t.get("subject") or cs.DEFAULT_SUBJECT)
130
+ try:
131
+ subject = subject.format(customer=row["Customer"], company=cs.COMPANY, month=month)
132
+ except Exception: # noqa: BLE001
133
+ # An unknown placeholder is the user's typo, not a 500. Show the template verbatim so they
134
+ # can see what they typed rather than getting an opaque error.
135
+ # ⛔ W36-T42: THIS CAUGHT THREE OF THE WAYS `str.format` FAILS AND THERE ARE MORE.
136
+ # `{customer` is a ValueError, `{customer.x}` an AttributeError, `{customer:%Y}` a
137
+ # TypeError, and every one of them is the same user mistake this branch was written for.
138
+ # Naming a subset of the exception types turns a typo in the OTHER half into a 500 on the
139
+ # one screen somebody opens to avoid mailing 200 people the wrong thing.
140
+ pass
141
+ return {
142
+ "html": cs.render_statement_html(row, t.get("intro") or cs.DEFAULT_INTRO,
143
+ t.get("footer") or cs.DEFAULT_FOOTER),
144
+ "to": row.get("Email") or "",
145
+ "subject": subject,
146
+ }
147
+
148
+
149
+ @router.post("/admin/statements/send")
150
+ def send(body: dict = Body(default=None), session: Session = Depends(_gate)):
151
+ """Queue statements. Returns per-customer outcomes — NEVER a bare count.
152
+
153
+ `overrideTo` is the test-send path: one customer, one address. SAFE_MODE still applies (the
154
+ data layer refuses an address outside the allow-list), which is why the route does not check it.
155
+ """
156
+ cs = _cs()
157
+ body = body or {}
158
+ names = [str(n) for n in (body.get("customers") or []) if str(n).strip()]
159
+ if not names:
160
+ raise err(400, "bad_request", "name at least one customer")
161
+ # ⛔⛔ W36-T42 / D-299 — A DECLARED TEST SEND WITH NO ADDRESS REFUSES. IT DOES NOT BECOME A
162
+ # REAL ONE. This read `(body.get("overrideTo") or "").strip() or None`, so a caller who sent
163
+ # `overrideTo: ""` (the field left blank, the state not yet typed into, a trimmed-away space)
164
+ # got a REAL statement mailed to the debtor's own address, reported back as `test: false`. On
165
+ # the one route in this product that is allowed to write to Odoo, the difference between a
166
+ # rehearsal and mailing a live customer was one empty string.
167
+ #
168
+ # ⭐ THE DISCRIMINATOR IS THE CALLER'S DECLARATION, NOT THE VALUE. A body with no `overrideTo`
169
+ # key and no `test` flag is the PRODUCTION send, and refusing that would delete the feature
170
+ # D-299 exists to preserve. What can be refused is a caller who SAID this is a test: the key
171
+ # being present, or `test: true`, is that statement, and an empty address beside it is the
172
+ # mistake. So both spellings of "absent" are covered: the key present and blank, and `test`
173
+ # asserted with no key at all.
174
+ #
175
+ # ⚠ THIS IS THE SECOND OF THREE WALLS AND THE ONLY ONE A PAYLOAD CANNOT ROUTE AROUND.
176
+ # `automationApi.testSendStatement` refuses an empty address before the request is built, and
177
+ # SAFE_MODE refuses an address outside the allow-list inside `queue_statement`. The client
178
+ # wall is bypassable by construction (anything can POST); this one is not.
179
+ declared_test = ("overrideTo" in body) or bool(body.get("test"))
180
+ override = str(body.get("overrideTo") or "").strip()
181
+ if declared_test and not override:
182
+ raise err(400, "no_override_address",
183
+ "a test send needs the address to send the test to. Without one this would "
184
+ "mail the customer's own address, which is the opposite of a test")
185
+ override = override or None
186
+ if override and len(names) != 1:
187
+ raise err(400, "bad_request", "a test send takes exactly one customer")
188
+ t = body.get("templates") or {}
189
+ rows, _ = _rows()
190
+
191
+ sent, failed, skipped = [], [], []
192
+ for name in names:
193
+ row = _find(rows, name)
194
+ if row is None:
195
+ failed.append({"customer": name, "error": "not on the current collection list"})
196
+ continue
197
+ if not override and not row.get("Email"):
198
+ # The Streamlit page warned and skipped these. Reporting them SEPARATELY from failures
199
+ # keeps "we could not" distinct from "there was nowhere to send".
200
+ skipped.append({"customer": name, "reason": "no email address on the customer record"})
201
+ continue
202
+ try:
203
+ mid = cs.queue_statement(cs.Odoo(), row, t.get("subject") or cs.DEFAULT_SUBJECT,
204
+ t.get("intro") or cs.DEFAULT_INTRO,
205
+ t.get("footer") or cs.DEFAULT_FOOTER, override_to=override)
206
+ sent.append({"customer": name, "to": override or row.get("Email"), "mailId": mid})
207
+ except Exception as e:
208
+ # SafeModeBlocked lands here too, and that is correct: to the caller a guardrail refusal
209
+ # and an Odoo error are both "this one did not go", each with its own honest message.
210
+ failed.append({"customer": name, "error": str(e)[:200]})
211
+ return {"sent": sent, "failed": failed, "skipped": skipped,
212
+ "safeMode": bool(cs.SAFE_MODE), "test": bool(override)}
213
+
214
+
215
+ # ---------------------------------------------------------------------------------------------
216
+ # WAVE 35 · T36 / CONTRACT C8 / OWNER RULING R10 — THE AGENT'S PARKED BATCH.
217
+ #
218
+ # ⛔⛔ THE AGENT ASSEMBLES; A PERSON CLICKS SEND. `automation_engine.run_statements` renders a batch
219
+ # and parks it on the automation definition, and it imports no mail path at all. THIS is the only
220
+ # door that releases one, and it is deliberately built out of the parts already here:
221
+ # · `_gate` — `admin_gate` THEN `_royal_only`, byte-for-byte the dependency the other three
222
+ # endpoints use. Not a copy of the checks: the same object.
223
+ # · `cs.queue_statement` — the same call `send()` above makes, so SAFE_MODE's allow-list refusal
224
+ # happens in the DATA LAYER, where no route, payload or UI can reach around it.
225
+ # ⇒ Nothing about the guardrails is re-expressed here, which is what makes "unchanged" checkable.
226
+ #
227
+ # ⚠ WHY THE BATCH IS RE-RENDERED FROM THE LIVE WORKLIST RATHER THAN SENT AS STORED. The parked
228
+ # `html` is what a person APPROVED and is what the review screen shows; but a balance can move
229
+ # between the parking and the click, and mailing a figure we know to be stale is worse than mailing
230
+ # a fresh one. So the parked batch decides WHO and WITH WHAT WORDS, and the live row decides the
231
+ # NUMBERS — the same split `preview` already makes. A customer who has left the worklist entirely
232
+ # (they paid) is reported as skipped rather than invoiced.
233
+
234
+ @router.get("/admin/statements/agent/{auto_id}")
235
+ def agent_batch(auto_id: str, session: Session = Depends(_gate)):
236
+ """What is parked and waiting for a click, for ONE agent. `null` when nothing is."""
237
+ defn = (engine.all_definitions(session.runtime) or {}).get(str(auto_id)) or {}
238
+ pending = defn.get("pendingStatements")
239
+ if not isinstance(pending, dict):
240
+ return {"pending": None}
241
+ # ⚠ The rendered `html` is NOT returned in the list — a 200-statement batch of rendered mail is
242
+ # megabytes, and the review screen needs the count and the names to decide. `preview` already
243
+ # renders ONE on demand.
244
+ return {"pending": {
245
+ "ts": pending.get("ts"), "count": int(pending.get("count") or 0),
246
+ "notes": list(pending.get("notes") or []),
247
+ "items": [{k: v for k, v in it.items() if k != "html"}
248
+ for it in (pending.get("items") or [])],
249
+ "safeMode": bool(_cs().SAFE_MODE)}}
250
+
251
+
252
+ @router.post("/admin/statements/agent/{auto_id}/send")
253
+ def agent_send(auto_id: str, body: dict = Body(default=None),
254
+ session: Session = Depends(_gate)):
255
+ """Release a parked batch. THE CLICK R10 REQUIRES — nothing else in the system calls this.
256
+
257
+ ⛔ IT REFUSES AN EMPTY OR MISSING BATCH rather than answering 200 with nothing sent: a send
258
+ that reports success and mails nobody is the `view_upsert` failure mode (200 OK, zero writes)
259
+ arriving on the one route in this system that talks to a mail server.
260
+ """
261
+ cs = _cs()
262
+ defn = (engine.all_definitions(session.runtime) or {}).get(str(auto_id)) or {}
263
+ pending = defn.get("pendingStatements")
264
+ items = list((pending or {}).get("items") or []) if isinstance(pending, dict) else []
265
+ if not items:
266
+ raise err(404, "no_batch", "there is nothing parked for this agent to send")
267
+ # ⚠ An explicit subset is allowed (a person unticking a customer on the review screen) but it
268
+ # may only NARROW the parked batch. A name that was never parked cannot be introduced by the
269
+ # payload, or the review stops being what authorised the send.
270
+ want = {str(n) for n in (body or {}).get("customers") or []}
271
+ if want:
272
+ items = [it for it in items if str(it.get("customer")) in want]
273
+ if not items:
274
+ raise err(400, "bad_request", "none of those customers are in the parked batch")
275
+ rows, _ = _rows()
276
+ sent, failed, skipped = [], [], []
277
+ for it in items:
278
+ name = str(it.get("customer") or "")
279
+ row = _find(rows, name)
280
+ if row is None:
281
+ skipped.append({"customer": name,
282
+ "reason": "no longer on the collection list, so nothing is owed"})
283
+ continue
284
+ try:
285
+ # ⛔ THE SAME DATA-LAYER CALL `send()` MAKES. SafeModeBlocked is raised INSIDE
286
+ # `queue_statement`, so the guardrail cannot be argued with from here.
287
+ mid = cs.queue_statement(cs.Odoo(), row,
288
+ str(it.get("subject") or "") or cs.DEFAULT_SUBJECT,
289
+ cs.DEFAULT_INTRO, cs.DEFAULT_FOOTER)
290
+ sent.append({"customer": name, "to": row.get("Email"), "mailId": mid})
291
+ except Exception as e: # noqa: BLE001
292
+ failed.append({"customer": name, "error": str(e)[:200]})
293
+ # ⚠ CLEARED ONLY WHEN NOTHING IS LEFT TO RETRY. A batch dropped while some of it failed would
294
+ # lose the list of who still needs a statement, and nobody would know to look.
295
+ if sent and not failed:
296
+ engine.clear_statements(session.runtime, auto_id)
297
+ return {"sent": sent, "failed": failed, "skipped": skipped,
298
+ "safeMode": bool(cs.SAFE_MODE), "cleared": bool(sent and not failed)}
api/routes_tables.py CHANGED
The diff for this file is too large to render. See raw diff
 
api/routes_web_agent.py CHANGED
@@ -1,84 +1,84 @@
1
- """routes_web_agent.py — the door that lets a person TEST the web agent (wave 31, R10 / D-51).
2
-
3
- The owner's words are the whole reason this file exists: *"we already laid the foundation of this
4
- but never test anything."* The capability is otherwise reachable only from inside an automation
5
- run, which means the first person to discover it is broken is a customer at 3am.
6
-
7
- GET /api/v1/web-agent/capability any session — can this deployment run a web step, and
8
- if not, the SENTENCE saying why
9
- POST /api/v1/web-agent/test ADMIN — run one real `web_read` and show what
10
- came back, or the sentence
11
-
12
- ⛔ NEITHER ROUTE IS THE SEAM. `automation_engine` calls `web_agent.run_step` directly (contract
13
- C5); these are an operator surface over the same function, so a green test here and a red run
14
- there cannot disagree about anything except the input.
15
-
16
- ⚠ `POST /test` BLOCKS FOR ~10-30 s and COSTS A FRACTION OF A CENT. It is `def`, not `async def`,
17
- so FastAPI runs it in the threadpool and one test cannot stall the event loop for everybody.
18
-
19
- ⚠ ON THE URL IT WILL FETCH: the fetch happens inside an ephemeral HF Job on Hugging Face's
20
- network, never from this server, so this is not a door into our own infrastructure. It is still
21
- admin-gated, because it spends money and because D-51 §5's authorisation posture ("only systems
22
- the tenant is authorised to use, at their instruction") is not something an ordinary member
23
- should be able to commit the tenant to.
24
-
25
- ⭐ MOUNTED, and the gate is what says so rather than this sentence: `main.py:80` imports it and
26
- `main.py:316` includes it, and `verify_web_agent.py` asserts the route answers rather than trusting
27
- either line. This paragraph read "NOT MOUNTED YET" for a whole wave after the mount landed, which
28
- is the same class of stale claim as a green gate on a router nobody wired: three finished routers
29
- once shipped 404-dead behind entirely green gates, and prose is not the control that stops it.
30
- """
31
- from fastapi import APIRouter, Body, Depends
32
-
33
- import web_agent
34
- from deps import Session, require_session, err
35
- from routes_admin import admin_gate
36
-
37
- router = APIRouter(prefix="/api/v1")
38
-
39
- MAX_URL = 2000
40
- MAX_SELECTOR = 400
41
-
42
-
43
- @router.get("/web-agent/capability")
44
- def web_agent_capability(session: Session = Depends(require_session)):
45
- """Can a web step run here at all? Configuration, not liveness — see `web_agent.capability`.
46
-
47
- Deliberately NARROW: it answers the question a UI needs ("may I offer this, and what do I say
48
- if not") and withholds the deployment detail (namespace, which token key, the image) that
49
- only an operator has any use for. Nothing here is ever a credential.
50
- """
51
- cap = web_agent.capability()
52
- return {"ready": bool(cap["ready"]), "reason": cap["reason"],
53
- "runnableKinds": cap["runnableKinds"], "profile": cap["profile"]}
54
-
55
-
56
- @router.post("/web-agent/test")
57
- def web_agent_test(session: Session = Depends(admin_gate), body: dict = Body(...)):
58
- """Run ONE real `web_read` and report exactly what the seam returned.
59
-
60
- The response mirrors the seam's own contract rather than flattening it: `ok` plus a `value`,
61
- or `ok:false` plus the SENTENCE. A test surface that turns a named failure into "something
62
- went wrong" would hide the one thing it exists to show.
63
- """
64
- url = str(body.get("url") or "").strip()
65
- selector = str(body.get("selector") or "").strip()
66
- if len(url) > MAX_URL or len(selector) > MAX_SELECTOR:
67
- raise err(400, "too_long", "the URL or selector is longer than this door accepts")
68
-
69
- step = {"kind": "web_read", "id": "test", "url": url, "selector": selector,
70
- "attr": (body.get("attr") or "text"), "all": bool(body.get("all")),
71
- "timeoutMs": int(body.get("timeoutMs") or 20000)}
72
- if body.get("waitFor"):
73
- step["waitFor"] = str(body["waitFor"])
74
-
75
- notes = []
76
- result, error = web_agent.run_step(
77
- step, {"tenant": session.tenant, "runId": f"test-{session.tenant}-{session.uname}",
78
- "log": notes.append})
79
- if error:
80
- # 200 with `ok:false`, not a 4xx: the request was well-formed and the ANSWER is that the
81
- # web step did not succeed. A 500 here would make an ordinary "the selector matched
82
- # nothing" look like a server fault.
83
- return {"ok": False, "error": error, "log": notes[-6:]}
84
- return {"ok": True, "result": result, "log": notes[-6:]}
 
1
+ """routes_web_agent.py — the door that lets a person TEST the web agent (wave 31, R10 / D-51).
2
+
3
+ The owner's words are the whole reason this file exists: *"we already laid the foundation of this
4
+ but never test anything."* The capability is otherwise reachable only from inside an automation
5
+ run, which means the first person to discover it is broken is a customer at 3am.
6
+
7
+ GET /api/v1/web-agent/capability any session — can this deployment run a web step, and
8
+ if not, the SENTENCE saying why
9
+ POST /api/v1/web-agent/test ADMIN — run one real `web_read` and show what
10
+ came back, or the sentence
11
+
12
+ ⛔ NEITHER ROUTE IS THE SEAM. `automation_engine` calls `web_agent.run_step` directly (contract
13
+ C5); these are an operator surface over the same function, so a green test here and a red run
14
+ there cannot disagree about anything except the input.
15
+
16
+ ⚠ `POST /test` BLOCKS FOR ~10-30 s and COSTS A FRACTION OF A CENT. It is `def`, not `async def`,
17
+ so FastAPI runs it in the threadpool and one test cannot stall the event loop for everybody.
18
+
19
+ ⚠ ON THE URL IT WILL FETCH: the fetch happens inside an ephemeral HF Job on Hugging Face's
20
+ network, never from this server, so this is not a door into our own infrastructure. It is still
21
+ admin-gated, because it spends money and because D-51 §5's authorisation posture ("only systems
22
+ the tenant is authorised to use, at their instruction") is not something an ordinary member
23
+ should be able to commit the tenant to.
24
+
25
+ ⭐ MOUNTED, and the gate is what says so rather than this sentence: `main.py:80` imports it and
26
+ `main.py:316` includes it, and `verify_web_agent.py` asserts the route answers rather than trusting
27
+ either line. This paragraph read "NOT MOUNTED YET" for a whole wave after the mount landed, which
28
+ is the same class of stale claim as a green gate on a router nobody wired: three finished routers
29
+ once shipped 404-dead behind entirely green gates, and prose is not the control that stops it.
30
+ """
31
+ from fastapi import APIRouter, Body, Depends
32
+
33
+ import web_agent
34
+ from deps import Session, require_session, err
35
+ from routes_admin import admin_gate
36
+
37
+ router = APIRouter(prefix="/api/v1")
38
+
39
+ MAX_URL = 2000
40
+ MAX_SELECTOR = 400
41
+
42
+
43
+ @router.get("/web-agent/capability")
44
+ def web_agent_capability(session: Session = Depends(require_session)):
45
+ """Can a web step run here at all? Configuration, not liveness — see `web_agent.capability`.
46
+
47
+ Deliberately NARROW: it answers the question a UI needs ("may I offer this, and what do I say
48
+ if not") and withholds the deployment detail (namespace, which token key, the image) that
49
+ only an operator has any use for. Nothing here is ever a credential.
50
+ """
51
+ cap = web_agent.capability()
52
+ return {"ready": bool(cap["ready"]), "reason": cap["reason"],
53
+ "runnableKinds": cap["runnableKinds"], "profile": cap["profile"]}
54
+
55
+
56
+ @router.post("/web-agent/test")
57
+ def web_agent_test(session: Session = Depends(admin_gate), body: dict = Body(...)):
58
+ """Run ONE real `web_read` and report exactly what the seam returned.
59
+
60
+ The response mirrors the seam's own contract rather than flattening it: `ok` plus a `value`,
61
+ or `ok:false` plus the SENTENCE. A test surface that turns a named failure into "something
62
+ went wrong" would hide the one thing it exists to show.
63
+ """
64
+ url = str(body.get("url") or "").strip()
65
+ selector = str(body.get("selector") or "").strip()
66
+ if len(url) > MAX_URL or len(selector) > MAX_SELECTOR:
67
+ raise err(400, "too_long", "the URL or selector is longer than this door accepts")
68
+
69
+ step = {"kind": "web_read", "id": "test", "url": url, "selector": selector,
70
+ "attr": (body.get("attr") or "text"), "all": bool(body.get("all")),
71
+ "timeoutMs": int(body.get("timeoutMs") or 20000)}
72
+ if body.get("waitFor"):
73
+ step["waitFor"] = str(body["waitFor"])
74
+
75
+ notes = []
76
+ result, error = web_agent.run_step(
77
+ step, {"tenant": session.tenant, "runId": f"test-{session.tenant}-{session.uname}",
78
+ "log": notes.append})
79
+ if error:
80
+ # 200 with `ok:false`, not a 4xx: the request was well-formed and the ANSWER is that the
81
+ # web step did not succeed. A 500 here would make an ordinary "the selector matched
82
+ # nothing" look like a server fault.
83
+ return {"ok": False, "error": error, "log": notes[-6:]}
84
+ return {"ok": True, "result": result, "log": notes[-6:]}
platform/aios_grid.py CHANGED
The diff for this file is too large to render. See raw diff
 
platform/aios_grid_fields.json CHANGED
@@ -1,581 +1,581 @@
1
- {
2
- "_comment": "CANONICAL field contract for the AIOS Airtable-style grid — the SINGLE source of truth. Consumed by platform/aios_grid.py (embed/Space host) and aios-web/api/main.py (standalone API), and regenerated into aios-web/web/public/sample_customers.json. Edit HERE only, then run aios-web/verify_fields_contract.py. source=odoo is READ-ONLY; source=overlay is the editable stratum (notes/tags) outside Odoo. type in {text,status,select,currency,int,date,pct} (select = a fixed-choice READ-ONLY brand attribute; dba is the first, wave 2026-08-02). `description` (wave 5) is the CANONICAL per-field description — every field must carry one, and since wave 7 (owner W8, 2026-07-28) every description is ONE SHORT PLAIN sentence (two only when a fact would otherwise mislead): what the field IS, nothing else — no filter tips, no '(none)' coaching, no rationale; the user's workspace NOTE overrides it in the (i) hover, never in this file. BUILDER FACT (documented here, deliberately NOT in user-facing text): blank text attributes display as '(none)', so `is '(none)'` — not `is empty` — finds the blanks on agent/city/state/country/zip/payment_terms/pricelist/tags. filterable:false = the CONDITION BUILDER does not offer it (still displayed, still sortable); every such field must have a replacement declared in aios-web/verify_fields_contract.py. 2026-07-27 partner attributes: country/zip/payment_terms/pricelist/tags/customer_since all ship default:false. zip is TEXT because a postal code has leading zeros. Odoo's credit_limit (1% populated) and user_id salesperson (2%) are deliberately ABSENT; agent_ids is the salesperson field and AR is where credit exposure comes from. Wave-5 item 8 (2026-07-27): ltm_rev and at_risk are DELETED — LTM's replacement is a creatable Sales measure column (the demo column IS Sales · the last 12 months), at_risk's replacement is a formula field, e.g. MAX(0, {revenue_ly} - {revenue_ytd}). Wave-6 item 8 (2026-07-27, the no-buildable-presets rule): revenue_ytd, revenue_ly, orders_24m, aov and yoy_pct are DELETED — every one is self-buildable, so a frozen pre-set beside the builder was two ways to ask one question. Replacements (recorded in verify_fields_contract.py): creatable measure columns for Sales / Orders / Avg order $ over any period (harness/measure_filter.py ADMITTED carries revenue, orders and the composite aov), and a formula over two measure columns for YoY, e.g. ({sales_ytd} - {sales_ly}) / {sales_ly}. Stale view colIds naming the five self-heal on the next autosave (the established rule).",
3
- "fields": [
4
- {
5
- "key": "customer",
6
- "label": "Customer",
7
- "type": "text",
8
- "source": "odoo",
9
- "pinned": true,
10
- "default": true,
11
- "description": "The customer's name in Odoo. One row per customer who ordered in the last 24 months."
12
- },
13
- {
14
- "key": "partner_id",
15
- "label": "Odoo ID",
16
- "type": "int",
17
- "source": "odoo",
18
- "derived": true,
19
- "default": false,
20
- "description": "The Odoo res.partner id — the key every Odoo document joins on. DERIVED: this row's pid IS the partner id, so a stored copy would be a second source."
21
- },
22
- {
23
- "key": "odoo_status",
24
- "label": "Odoo record",
25
- "type": "status",
26
- "source": "odoo",
27
- "default": false,
28
- "options": [
29
- "Active",
30
- "Archived"
31
- ],
32
- "description": "Whether this customer still exists in Odoo. Archived means deleted there."
33
- },
34
- {
35
- "key": "agent",
36
- "label": "Agent",
37
- "type": "text",
38
- "source": "odoo",
39
- "default": true,
40
- "description": "The sales agent who owns this account."
41
- },
42
- {
43
- "key": "dba",
44
- "label": "DBA",
45
- "type": "select",
46
- "source": "odoo",
47
- "default": false,
48
- "options": [
49
- "Fisch",
50
- "Royal",
51
- "Both"
52
- ],
53
- "description": "The brand this customer buys from - Fisch, Royal, or both. Amazon-channel orders are not a DBA."
54
- },
55
- {
56
- "key": "salesperson",
57
- "label": "Salesperson",
58
- "type": "text",
59
- "source": "odoo",
60
- "default": false,
61
- "description": "Who keyed in most of this customer's orders — not the Agent, who owns the account."
62
- },
63
- {
64
- "key": "street",
65
- "label": "Street",
66
- "type": "text",
67
- "source": "odoo",
68
- "default": false,
69
- "description": "First address line, from res.partner directly - not the geocoder, so a customer the map cannot place still shows its address."
70
- },
71
- {
72
- "key": "street2",
73
- "label": "Street 2",
74
- "type": "text",
75
- "source": "odoo",
76
- "default": false,
77
- "description": "Second address line (suite, unit, floor) on the customer's Odoo address."
78
- },
79
- {
80
- "key": "city",
81
- "label": "City",
82
- "type": "text",
83
- "source": "odoo",
84
- "default": true,
85
- "description": "City on the customer's Odoo address."
86
- },
87
- {
88
- "key": "state",
89
- "label": "State",
90
- "type": "text",
91
- "source": "odoo",
92
- "default": true,
93
- "description": "State or province on the customer's Odoo address."
94
- },
95
- {
96
- "key": "country",
97
- "label": "Country",
98
- "type": "text",
99
- "source": "odoo",
100
- "default": false,
101
- "description": "Country on the customer's Odoo address."
102
- },
103
- {
104
- "key": "zip",
105
- "label": "ZIP",
106
- "type": "text",
107
- "source": "odoo",
108
- "default": false,
109
- "description": "Postal code on the customer's Odoo address."
110
- },
111
- {
112
- "key": "customer_since",
113
- "label": "Customer since",
114
- "type": "date",
115
- "source": "odoo",
116
- "default": false,
117
- "description": "When this customer was first set up in Odoo."
118
- },
119
- {
120
- "key": "tags",
121
- "label": "Tags",
122
- "type": "text",
123
- "source": "odoo",
124
- "default": false,
125
- "description": "Odoo labels on this customer, comma-separated."
126
- },
127
- {
128
- "key": "pricelist",
129
- "label": "Price list",
130
- "type": "text",
131
- "source": "odoo",
132
- "default": false,
133
- "description": "The price list this customer buys on."
134
- },
135
- {
136
- "key": "payment_terms",
137
- "label": "Payment terms",
138
- "type": "text",
139
- "source": "odoo",
140
- "default": false,
141
- "description": "Payment terms on this customer's account — Net 30, for example."
142
- },
143
- {
144
- "key": "last_order",
145
- "label": "Last order",
146
- "type": "date",
147
- "source": "odoo",
148
- "default": true,
149
- "description": "Date of the most recent confirmed order."
150
- },
151
- {
152
- "key": "overdue_days",
153
- "label": "Overdue days",
154
- "type": "int",
155
- "source": "odoo",
156
- "default": true,
157
- "description": "How many days late this customer is running against their own usual ordering rhythm."
158
- },
159
- {
160
- "_note": "filterable:false — DERIVED ANALYTIC: est_missed is min(cycles missed, 3) x AOV, a score we compute rather than an object the business has, so a condition on it would read as a fact about the customer when it is a fact about our arithmetic. It still displays and still sorts. Until wave 6 this flag also covered the frozen-window presets (revenue_ytd / revenue_ly / orders_24m / aov / yoy_pct); those are now DELETED outright under the owner's no-buildable-presets rule — see _comment. est_missed itself STAYS: no creatable measure or formula reproduces the cadence model behind it.",
161
- "key": "est_missed",
162
- "label": "Est. missed $",
163
- "type": "currency",
164
- "source": "odoo",
165
- "default": true,
166
- "agg": "sum",
167
- "filterable": false,
168
- "description": "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed."
169
- },
170
- {
171
- "_note": "wave 21 R1 — KEY UNCHANGED, LABEL RENAMED. The computation is a DISJOINT split (ar.py credit_exposure): this column is only the not-yet-due residual, its sibling is the past-grace residual, and the two sum to the total. Under the label 'AR open $' the majority-late book read as 'Overdue > Open', which is nonsense in AR vocabulary — 'open' universally means the total. The label now says what the number is; the key stays so saved views and filters keep working.",
172
- "key": "ar_open",
173
- "label": "AR current $",
174
- "type": "currency",
175
- "source": "odoo",
176
- "default": false,
177
- "description": "Invoiced money owed but not yet due (a 5-day grace applies before it counts as overdue)."
178
- },
179
- {
180
- "key": "ar_overdue",
181
- "label": "AR overdue $",
182
- "type": "currency",
183
- "source": "odoo",
184
- "default": false,
185
- "description": "Invoiced money past due — same basis as the Collections page."
186
- },
187
- {
188
- "_note": "wave 21 R1 — the TOTAL, added beside the rename above. AR current $ + AR overdue $, i.e. what most people mean by 'open AR'. Composed from the same ar.credit_exposure rows the siblings use, so it is transitively reconciled by ar.validate()'s residual read_group tie — no second oracle.",
189
- "key": "ar_outstanding",
190
- "label": "AR outstanding $",
191
- "type": "currency",
192
- "source": "odoo",
193
- "default": false,
194
- "description": "Total invoiced money owed right now: AR current $ plus AR overdue $."
195
- },
196
- {
197
- "key": "ar_exposure",
198
- "label": "Credit exposure $",
199
- "type": "currency",
200
- "source": "odoo",
201
- "default": false,
202
- "description": "The most you could be out if they stopped paying today: open, overdue, draft and not-yet-invoiced."
203
- },
204
- {
205
- "key": "ar_aged_1_30",
206
- "label": "1-30 days $",
207
- "type": "currency",
208
- "source": "odoo",
209
- "default": false,
210
- "description": "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $."
211
- },
212
- {
213
- "key": "ar_aged_31_60",
214
- "label": "31-60 days $",
215
- "type": "currency",
216
- "source": "odoo",
217
- "default": false,
218
- "description": "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $."
219
- },
220
- {
221
- "key": "ar_aged_61_90",
222
- "label": "61-90 days $",
223
- "type": "currency",
224
- "source": "odoo",
225
- "default": false,
226
- "description": "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $."
227
- },
228
- {
229
- "key": "ar_aged_90_plus",
230
- "label": "90+ days $",
231
- "type": "currency",
232
- "source": "odoo",
233
- "default": false,
234
- "description": "Overdue by more than 90 days. The four aging buckets sum to AR overdue $."
235
- },
236
- {
237
- "key": "days_to_pay",
238
- "label": "Days to pay",
239
- "type": "int",
240
- "source": "odoo",
241
- "default": false,
242
- "description": "Average days to pay an invoice in full. Blank means no fully paid invoice yet."
243
- },
244
- {
245
- "key": "top_category",
246
- "label": "Top category",
247
- "type": "text",
248
- "source": "odoo",
249
- "default": false,
250
- "description": "The category this customer spent the most on in the last 12 months."
251
- },
252
- {
253
- "key": "top_category_pct",
254
- "label": "Top category %",
255
- "type": "pct",
256
- "source": "odoo",
257
- "default": false,
258
- "description": "Share of last-12-months spend that went to the top category."
259
- },
260
- {
261
- "key": "sku_count",
262
- "label": "SKUs bought",
263
- "type": "int",
264
- "source": "odoo",
265
- "default": false,
266
- "description": "Distinct products bought in the last 12 months."
267
- },
268
- {
269
- "key": "top_sku",
270
- "label": "Top SKU",
271
- "type": "text",
272
- "source": "odoo",
273
- "default": false,
274
- "description": "The product this customer spent the most on in the last 12 months."
275
- },
276
- {
277
- "key": "days_since",
278
- "label": "Days since order",
279
- "type": "int",
280
- "source": "odoo",
281
- "default": false,
282
- "description": "Days since the last confirmed order."
283
- },
284
- {
285
- "key": "typical_gap_days",
286
- "label": "Typical gap days",
287
- "type": "int",
288
- "source": "odoo",
289
- "default": false,
290
- "description": "Days this customer usually goes between orders, from their own history."
291
- },
292
- {
293
- "key": "notes",
294
- "label": "Notes",
295
- "type": "text",
296
- "source": "overlay",
297
- "default": false,
298
- "description": "Your notes on this customer. Saved in this app only, visible only to you."
299
- }
300
- ],
301
- "_product_comment": "ADDITIVE, wave 15 C-TOPIC. The PRODUCT table's field contract. Kept as a SEPARATE top-level key rather than restructuring `fields` into {customer_data, product_data}: both existing readers (aios_grid._load_fields, aios-web/api/main.py) index doc['fields'] directly, and reshaping that mid-wave would break the embed for a cosmetic gain. The keyed shape can arrive when both readers move in ONE commit; until then this is the product half and `fields` is the customer half.",
302
- "_product_removed_buy_now": "OWNER, 2026-08-03: 'Buy signal' (key buy_now, a select of Buy now / OK) is NO LONGER A PRESET FIELD. It never earned one: it is a formula over two columns that are both still right here, and the platform has a formula field type for exactly that. THE FORMULA, which reproduces the retired column row for row (modules/product_data.validate proves the equivalence, and goes red if it ever stops holding): IF({lead_days} > 0, IF({dos} < {lead_days}, \"Buy now\", \"OK\"), \"\") . Every branch matches the old server rule, including the blanks - the formula engine refuses a comparison against a blank rather than coercing it to 0, so a SKU with no days-of-supply or no lead time comes out empty, which is 'we do not know' and not 'you are fine'. NOTE the column is still COMPUTED in product_data.pool(): it ships nowhere (rows_from_pool projects strictly through this contract, so no Field means no cell on the wire) and exists only as validate()'s oracle. A formula field is PER-USER, so nothing shared may filter on it - the Buy list view filters on dos/lead_days directly (_seed_wave17).",
303
- "product_data": {
304
- "identity": "pid",
305
- "business_key": "code",
306
- "fields": [
307
- {
308
- "key": "code",
309
- "label": "SKU",
310
- "type": "text",
311
- "source": "odoo",
312
- "pinned": true,
313
- "default": true,
314
- "description": "The SKU code — the product's real business key. `pid` is a stable CRC32 of it because the grid keys on an integer."
315
- },
316
- {
317
- "key": "product",
318
- "label": "Product",
319
- "type": "text",
320
- "source": "odoo",
321
- "default": true,
322
- "description": "Product name as it appears in Odoo."
323
- },
324
- {
325
- "key": "category",
326
- "label": "Category",
327
- "type": "select",
328
- "source": "odoo",
329
- "default": true,
330
- "description": "Product category; '(uncategorized)' when Odoo carries none."
331
- },
332
- {
333
- "key": "supplier",
334
- "label": "Supplier",
335
- "type": "text",
336
- "source": "overlay",
337
- "default": true,
338
- "description": "Who makes it. Editable here and shared with everyone in the workspace; seeded from the inventory mastersheet.",
339
- "shared": true
340
- },
341
- {
342
- "key": "origin_country",
343
- "label": "Country",
344
- "type": "text",
345
- "source": "overlay",
346
- "default": false,
347
- "description": "Country of origin. Editable here and shared with everyone; seeded from the inventory mastersheet.",
348
- "shared": true
349
- },
350
- {
351
- "key": "lead_days",
352
- "label": "Lead time (days)",
353
- "type": "int",
354
- "source": "overlay",
355
- "default": true,
356
- "description": "Order-to-arrival days for this supplier. Drives the buy signal. Editable and shared with everyone.",
357
- "shared": true
358
- },
359
- {
360
- "key": "first_cost",
361
- "label": "First cost",
362
- "type": "currency",
363
- "source": "overlay",
364
- "default": false,
365
- "description": "Quoted unit cost at origin, before freight and duty. Editable and shared with everyone.",
366
- "shared": true
367
- },
368
- {
369
- "key": "price_fisch",
370
- "label": "Fisch price",
371
- "type": "currency",
372
- "source": "odoo",
373
- "description": "Fisch pricelist price for this SKU. Blank when that list prices it nowhere."
374
- },
375
- {
376
- "key": "price_royal_1",
377
- "label": "Royal 1 price",
378
- "type": "currency",
379
- "source": "odoo",
380
- "description": "Royal 1 pricelist price for this SKU. Blank when that list prices it nowhere."
381
- },
382
- {
383
- "key": "price_royal_2",
384
- "label": "Royal 2 price",
385
- "type": "currency",
386
- "source": "odoo",
387
- "description": "Royal 2 pricelist price for this SKU. Blank when that list prices it nowhere."
388
- },
389
- {
390
- "key": "tier_prices",
391
- "label": "Tier prices",
392
- "type": "json",
393
- "source": "odoo",
394
- "description": "Every live pricelist that prices this SKU today, as [{pricelist, unit_price}] at qty 1. Blank when no list prices it. This is the honest SET; the three Fisch/Royal columns beside it are the DECLARED subset and cannot show a price on a list the contract does not name."
395
- },
396
- {
397
- "key": "units",
398
- "label": "Units",
399
- "type": "json",
400
- "source": "odoo",
401
- "description": "The units of measure this SKU is really sold in, as [{name, qty}] where qty is in the product's own unit. Blank means it is sold in ONE unit, not that data is missing - only 1,150 of 5,873 active SKUs (19.6%) carry a unit tier."
402
- },
403
- {
404
- "key": "rev_ytd",
405
- "label": "Revenue YTD",
406
- "type": "currency",
407
- "source": "odoo",
408
- "default": true,
409
- "description": "Year-to-date revenue for this SKU, BU-scoped when the caller is."
410
- },
411
- {
412
- "key": "rev_ly",
413
- "label": "Revenue LY",
414
- "type": "currency",
415
- "source": "odoo",
416
- "description": "Same period last year — seasonal wholesale compares like for like."
417
- },
418
- {
419
- "key": "yoy_pct",
420
- "label": "YoY %",
421
- "type": "pct",
422
- "source": "odoo",
423
- "description": "Year-over-year change; null when last year was zero (a ratio to zero is not a number)."
424
- },
425
- {
426
- "key": "qty_ytd",
427
- "label": "Units YTD",
428
- "type": "int",
429
- "source": "odoo",
430
- "description": "Units sold year to date."
431
- },
432
- {
433
- "key": "orders_ytd",
434
- "label": "Orders YTD",
435
- "type": "int",
436
- "source": "odoo",
437
- "description": "Distinct orders containing this SKU, year to date."
438
- },
439
- {
440
- "key": "on_hand",
441
- "label": "On hand",
442
- "type": "int",
443
- "source": "odoo",
444
- "description": "Units in stock. CONSOLIDATED — one physical warehouse, not brand-tagged, so this column is ABSENT for a BU-scoped caller rather than silently company-wide."
445
- },
446
- {
447
- "key": "unit_cost",
448
- "label": "Unit cost",
449
- "type": "currency",
450
- "source": "odoo",
451
- "description": "Inventory unit cost. Consolidated; absent for a BU-scoped caller."
452
- },
453
- {
454
- "key": "inv_value",
455
- "label": "Stock value",
456
- "type": "currency",
457
- "source": "odoo",
458
- "description": "On-hand value at cost. Consolidated; absent for a BU-scoped caller."
459
- },
460
- {
461
- "key": "qty_ltm",
462
- "label": "Units LTM",
463
- "type": "int",
464
- "source": "odoo",
465
- "description": "Units sold in the last twelve months. Consolidated; absent for a BU-scoped caller."
466
- },
467
- {
468
- "key": "incoming",
469
- "label": "Inbound units",
470
- "type": "int",
471
- "source": "odoo",
472
- "description": "Units already on order and not yet received, from Odoo's own incoming quantity on the product. In the product's STOCK unit of measure, the same unit as On hand, so the two can be added. Days of supply and both cover gap columns all count this as stock (owner, 2026-08-19)."
473
- },
474
- {
475
- "key": "dos",
476
- "label": "Days of supply",
477
- "type": "int",
478
- "source": "odoo",
479
- "description": "Days of supply at the trailing twelve month rate, counting ON HAND PLUS INBOUND units as stock; null means it never sells through. Same stock figure as the cover gap columns, so the two cannot disagree about how much you have. They still divide by different rates: this one is the trailing average, the cover gap uses the forward 8 month forecast, so a seasonal SKU reads differently in each. Consolidated; absent for a BU-scoped caller."
480
- },
481
- {
482
- "key": "cover_gap_d",
483
- "label": "Cover gap (days)",
484
- "type": "int",
485
- "source": "odoo",
486
- "default": false,
487
- "description": "Days of cover minus supplier lead time. Negative means it runs out before a reorder lands. It counts INBOUND units as stock, exactly as Days of supply now does. The one thing it does differently: the burn rate is the FORWARD 8 month forecast rather than the trailing twelve month average, so a seasonal SKU reads differently here."
488
- },
489
- {
490
- "key": "cover_gap_units",
491
- "label": "Cover gap (units)",
492
- "type": "int",
493
- "source": "odoo",
494
- "default": false,
495
- "description": "The recommended reorder quantity: units of forecast demand over the lead time that on-hand plus inbound does not cover. POSITIVE means buy this many; negative is surplus units; blank means we cannot say, because the SKU has no lead time on file or no forecast demand. Rounded AWAY from zero, so a real shortfall never rounds down to nothing."
496
- },
497
- {
498
- "key": "demand_fwd",
499
- "label": "Forecast units (8 mo)",
500
- "type": "int",
501
- "source": "odoo",
502
- "description": "Units this SKU is expected to sell over the next 8 months, read as the units it actually sold in the same 8 calendar months one year ago. Seasonal on purpose: a flat annual average spreads Valentine's, Mother's Day and Christmas evenly across the year and understates the months buyers actually order for. A SKU too new to appear in that window falls back to its trailing twelve month rate scaled to 8 months. This is the burn rate both cover gap columns divide by, shown so the reorder quantity can be checked."
503
- },
504
- {
505
- "key": "stock_bucket",
506
- "label": "Stock status",
507
- "type": "select",
508
- "source": "odoo",
509
- "description": "Dead / excess / healthy bucket, from the same days of supply figure beside it, so it counts inbound units too. One exception on purpose: 'Out of stock' still keys on the REAL shelf, because that is a present tense fact somebody can walk into the warehouse and check. A row can honestly read 'Out of stock' with a days of supply beside it when the replenishment is on the water; Inbound units is why. Consolidated; absent for a BU-scoped caller."
510
- },
511
- {
512
- "key": "discontinued",
513
- "label": "Discontinued",
514
- "type": "select",
515
- "source": "odoo",
516
- "options": [
517
- "Yes",
518
- "No"
519
- ],
520
- "description": "Whether Odoo carries the Discontinued product tag on this SKU. Every row gets an explicit Yes or No rather than a blank, because a blank reads as an inactive condition in the filter engine and would silently WIDEN any view that filtered on it."
521
- },
522
- {
523
- "key": "needs_pricing",
524
- "label": "Needs pricing",
525
- "type": "select",
526
- "source": "overlay",
527
- "default": false,
528
- "options": [
529
- "Yes"
530
- ],
531
- "shared": true,
532
- "description": "Team-maintained. A SKU carries “Yes” when it appears on the NEEDS PRICING tab of the 2027 catalog workbook; no value means it is not on that list. Shared with everyone in the workspace — the whole point of R8 is that the TEAM's work lands in the system, not one importer's private column."
533
- },
534
- {
535
- "key": "march_pricelist",
536
- "label": "March pricelist",
537
- "type": "select",
538
- "source": "overlay",
539
- "default": false,
540
- "options": [
541
- "Yes"
542
- ],
543
- "shared": true,
544
- "description": "Team-maintained. A SKU carries “Yes” when it appears on the March Pricelist tab of the 2027 catalog workbook; no value means it is not on that list. Shared with everyone in the workspace — the whole point of R8 is that the TEAM's work lands in the system, not one importer's private column."
545
- },
546
- {
547
- "key": "price_changes",
548
- "label": "Price changes",
549
- "type": "select",
550
- "source": "overlay",
551
- "default": false,
552
- "options": [
553
- "Yes"
554
- ],
555
- "shared": true,
556
- "description": "Team-maintained. A SKU carries “Yes” when it appears on the Price Changes tab of the 2027 catalog workbook; no value means it is not on that list. Shared with everyone in the workspace — the whole point of R8 is that the TEAM's work lands in the system, not one importer's private column."
557
- },
558
- {
559
- "key": "closeouts",
560
- "label": "Closeouts",
561
- "type": "select",
562
- "source": "overlay",
563
- "default": false,
564
- "options": [
565
- "Yes"
566
- ],
567
- "shared": true,
568
- "description": "Team-maintained. A SKU carries “Yes” when it appears on the Closeouts tab of the 2027 catalog workbook; no value means it is not on that list. Shared with everyone in the workspace — the whole point of R8 is that the TEAM's work lands in the system, not one importer's private column."
569
- },
570
- {
571
- "key": "notes",
572
- "label": "Notes",
573
- "type": "text",
574
- "source": "overlay",
575
- "default": false,
576
- "shared": true,
577
- "description": "Team-maintained. The 2027 workbook's own non-Odoo columns (Product Description, Packing) folded into one field. Shared with everyone in the workspace."
578
- }
579
- ]
580
- }
581
- }
 
1
+ {
2
+ "_comment": "CANONICAL field contract for the AIOS Airtable-style grid — the SINGLE source of truth. Consumed by platform/aios_grid.py (embed/Space host) and aios-web/api/main.py (standalone API), and regenerated into aios-web/web/public/sample_customers.json. Edit HERE only, then run aios-web/verify_fields_contract.py. source=odoo is READ-ONLY; source=overlay is the editable stratum (notes/tags) outside Odoo. type in {text,status,select,currency,int,date,pct} (select = a fixed-choice READ-ONLY brand attribute; dba is the first, wave 2026-08-02). `description` (wave 5) is the CANONICAL per-field description — every field must carry one, and since wave 7 (owner W8, 2026-07-28) every description is ONE SHORT PLAIN sentence (two only when a fact would otherwise mislead): what the field IS, nothing else — no filter tips, no '(none)' coaching, no rationale; the user's workspace NOTE overrides it in the (i) hover, never in this file. BUILDER FACT (documented here, deliberately NOT in user-facing text): blank text attributes display as '(none)', so `is '(none)'` — not `is empty` — finds the blanks on agent/city/state/country/zip/payment_terms/pricelist/tags. filterable:false = the CONDITION BUILDER does not offer it (still displayed, still sortable); every such field must have a replacement declared in aios-web/verify_fields_contract.py. 2026-07-27 partner attributes: country/zip/payment_terms/pricelist/tags/customer_since all ship default:false. zip is TEXT because a postal code has leading zeros. Odoo's credit_limit (1% populated) and user_id salesperson (2%) are deliberately ABSENT; agent_ids is the salesperson field and AR is where credit exposure comes from. Wave-5 item 8 (2026-07-27): ltm_rev and at_risk are DELETED — LTM's replacement is a creatable Sales measure column (the demo column IS Sales · the last 12 months), at_risk's replacement is a formula field, e.g. MAX(0, {revenue_ly} - {revenue_ytd}). Wave-6 item 8 (2026-07-27, the no-buildable-presets rule): revenue_ytd, revenue_ly, orders_24m, aov and yoy_pct are DELETED — every one is self-buildable, so a frozen pre-set beside the builder was two ways to ask one question. Replacements (recorded in verify_fields_contract.py): creatable measure columns for Sales / Orders / Avg order $ over any period (harness/measure_filter.py ADMITTED carries revenue, orders and the composite aov), and a formula over two measure columns for YoY, e.g. ({sales_ytd} - {sales_ly}) / {sales_ly}. Stale view colIds naming the five self-heal on the next autosave (the established rule).",
3
+ "fields": [
4
+ {
5
+ "key": "customer",
6
+ "label": "Customer",
7
+ "type": "text",
8
+ "source": "odoo",
9
+ "pinned": true,
10
+ "default": true,
11
+ "description": "The customer's name in Odoo. One row per customer who ordered in the last 24 months."
12
+ },
13
+ {
14
+ "key": "partner_id",
15
+ "label": "Odoo ID",
16
+ "type": "int",
17
+ "source": "odoo",
18
+ "derived": true,
19
+ "default": false,
20
+ "description": "The Odoo res.partner id — the key every Odoo document joins on. DERIVED: this row's pid IS the partner id, so a stored copy would be a second source."
21
+ },
22
+ {
23
+ "key": "odoo_status",
24
+ "label": "Odoo record",
25
+ "type": "status",
26
+ "source": "odoo",
27
+ "default": false,
28
+ "options": [
29
+ "Active",
30
+ "Archived"
31
+ ],
32
+ "description": "Whether this customer still exists in Odoo. Archived means deleted there."
33
+ },
34
+ {
35
+ "key": "agent",
36
+ "label": "Agent",
37
+ "type": "text",
38
+ "source": "odoo",
39
+ "default": true,
40
+ "description": "The sales agent who owns this account."
41
+ },
42
+ {
43
+ "key": "dba",
44
+ "label": "DBA",
45
+ "type": "select",
46
+ "source": "odoo",
47
+ "default": false,
48
+ "options": [
49
+ "Fisch",
50
+ "Royal",
51
+ "Both"
52
+ ],
53
+ "description": "The brand this customer buys from - Fisch, Royal, or both. Amazon-channel orders are not a DBA."
54
+ },
55
+ {
56
+ "key": "salesperson",
57
+ "label": "Salesperson",
58
+ "type": "text",
59
+ "source": "odoo",
60
+ "default": false,
61
+ "description": "Who keyed in most of this customer's orders — not the Agent, who owns the account."
62
+ },
63
+ {
64
+ "key": "street",
65
+ "label": "Street",
66
+ "type": "text",
67
+ "source": "odoo",
68
+ "default": false,
69
+ "description": "First address line, from res.partner directly - not the geocoder, so a customer the map cannot place still shows its address."
70
+ },
71
+ {
72
+ "key": "street2",
73
+ "label": "Street 2",
74
+ "type": "text",
75
+ "source": "odoo",
76
+ "default": false,
77
+ "description": "Second address line (suite, unit, floor) on the customer's Odoo address."
78
+ },
79
+ {
80
+ "key": "city",
81
+ "label": "City",
82
+ "type": "text",
83
+ "source": "odoo",
84
+ "default": true,
85
+ "description": "City on the customer's Odoo address."
86
+ },
87
+ {
88
+ "key": "state",
89
+ "label": "State",
90
+ "type": "text",
91
+ "source": "odoo",
92
+ "default": true,
93
+ "description": "State or province on the customer's Odoo address."
94
+ },
95
+ {
96
+ "key": "country",
97
+ "label": "Country",
98
+ "type": "text",
99
+ "source": "odoo",
100
+ "default": false,
101
+ "description": "Country on the customer's Odoo address."
102
+ },
103
+ {
104
+ "key": "zip",
105
+ "label": "ZIP",
106
+ "type": "text",
107
+ "source": "odoo",
108
+ "default": false,
109
+ "description": "Postal code on the customer's Odoo address."
110
+ },
111
+ {
112
+ "key": "customer_since",
113
+ "label": "Customer since",
114
+ "type": "date",
115
+ "source": "odoo",
116
+ "default": false,
117
+ "description": "When this customer was first set up in Odoo."
118
+ },
119
+ {
120
+ "key": "tags",
121
+ "label": "Tags",
122
+ "type": "text",
123
+ "source": "odoo",
124
+ "default": false,
125
+ "description": "Odoo labels on this customer, comma-separated."
126
+ },
127
+ {
128
+ "key": "pricelist",
129
+ "label": "Price list",
130
+ "type": "text",
131
+ "source": "odoo",
132
+ "default": false,
133
+ "description": "The price list this customer buys on."
134
+ },
135
+ {
136
+ "key": "payment_terms",
137
+ "label": "Payment terms",
138
+ "type": "text",
139
+ "source": "odoo",
140
+ "default": false,
141
+ "description": "Payment terms on this customer's account — Net 30, for example."
142
+ },
143
+ {
144
+ "key": "last_order",
145
+ "label": "Last order",
146
+ "type": "date",
147
+ "source": "odoo",
148
+ "default": true,
149
+ "description": "Date of the most recent confirmed order."
150
+ },
151
+ {
152
+ "key": "overdue_days",
153
+ "label": "Overdue days",
154
+ "type": "int",
155
+ "source": "odoo",
156
+ "default": true,
157
+ "description": "How many days late this customer is running against their own usual ordering rhythm."
158
+ },
159
+ {
160
+ "_note": "filterable:false — DERIVED ANALYTIC: est_missed is min(cycles missed, 3) x AOV, a score we compute rather than an object the business has, so a condition on it would read as a fact about the customer when it is a fact about our arithmetic. It still displays and still sorts. Until wave 6 this flag also covered the frozen-window presets (revenue_ytd / revenue_ly / orders_24m / aov / yoy_pct); those are now DELETED outright under the owner's no-buildable-presets rule — see _comment. est_missed itself STAYS: no creatable measure or formula reproduces the cadence model behind it.",
161
+ "key": "est_missed",
162
+ "label": "Est. missed $",
163
+ "type": "currency",
164
+ "source": "odoo",
165
+ "default": true,
166
+ "agg": "sum",
167
+ "filterable": false,
168
+ "description": "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed."
169
+ },
170
+ {
171
+ "_note": "wave 21 R1 — KEY UNCHANGED, LABEL RENAMED. The computation is a DISJOINT split (ar.py credit_exposure): this column is only the not-yet-due residual, its sibling is the past-grace residual, and the two sum to the total. Under the label 'AR open $' the majority-late book read as 'Overdue > Open', which is nonsense in AR vocabulary — 'open' universally means the total. The label now says what the number is; the key stays so saved views and filters keep working.",
172
+ "key": "ar_open",
173
+ "label": "AR current $",
174
+ "type": "currency",
175
+ "source": "odoo",
176
+ "default": false,
177
+ "description": "Invoiced money owed but not yet due (a 5-day grace applies before it counts as overdue)."
178
+ },
179
+ {
180
+ "key": "ar_overdue",
181
+ "label": "AR overdue $",
182
+ "type": "currency",
183
+ "source": "odoo",
184
+ "default": false,
185
+ "description": "Invoiced money past due — same basis as the Collections page."
186
+ },
187
+ {
188
+ "_note": "wave 21 R1 — the TOTAL, added beside the rename above. AR current $ + AR overdue $, i.e. what most people mean by 'open AR'. Composed from the same ar.credit_exposure rows the siblings use, so it is transitively reconciled by ar.validate()'s residual read_group tie — no second oracle.",
189
+ "key": "ar_outstanding",
190
+ "label": "AR outstanding $",
191
+ "type": "currency",
192
+ "source": "odoo",
193
+ "default": false,
194
+ "description": "Total invoiced money owed right now: AR current $ plus AR overdue $."
195
+ },
196
+ {
197
+ "key": "ar_exposure",
198
+ "label": "Credit exposure $",
199
+ "type": "currency",
200
+ "source": "odoo",
201
+ "default": false,
202
+ "description": "The most you could be out if they stopped paying today: open, overdue, draft and not-yet-invoiced."
203
+ },
204
+ {
205
+ "key": "ar_aged_1_30",
206
+ "label": "1-30 days $",
207
+ "type": "currency",
208
+ "source": "odoo",
209
+ "default": false,
210
+ "description": "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $."
211
+ },
212
+ {
213
+ "key": "ar_aged_31_60",
214
+ "label": "31-60 days $",
215
+ "type": "currency",
216
+ "source": "odoo",
217
+ "default": false,
218
+ "description": "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $."
219
+ },
220
+ {
221
+ "key": "ar_aged_61_90",
222
+ "label": "61-90 days $",
223
+ "type": "currency",
224
+ "source": "odoo",
225
+ "default": false,
226
+ "description": "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $."
227
+ },
228
+ {
229
+ "key": "ar_aged_90_plus",
230
+ "label": "90+ days $",
231
+ "type": "currency",
232
+ "source": "odoo",
233
+ "default": false,
234
+ "description": "Overdue by more than 90 days. The four aging buckets sum to AR overdue $."
235
+ },
236
+ {
237
+ "key": "days_to_pay",
238
+ "label": "Days to pay",
239
+ "type": "int",
240
+ "source": "odoo",
241
+ "default": false,
242
+ "description": "Average days to pay an invoice in full. Blank means no fully paid invoice yet."
243
+ },
244
+ {
245
+ "key": "top_category",
246
+ "label": "Top category",
247
+ "type": "text",
248
+ "source": "odoo",
249
+ "default": false,
250
+ "description": "The category this customer spent the most on in the last 12 months."
251
+ },
252
+ {
253
+ "key": "top_category_pct",
254
+ "label": "Top category %",
255
+ "type": "pct",
256
+ "source": "odoo",
257
+ "default": false,
258
+ "description": "Share of last-12-months spend that went to the top category."
259
+ },
260
+ {
261
+ "key": "sku_count",
262
+ "label": "SKUs bought",
263
+ "type": "int",
264
+ "source": "odoo",
265
+ "default": false,
266
+ "description": "Distinct products bought in the last 12 months."
267
+ },
268
+ {
269
+ "key": "top_sku",
270
+ "label": "Top SKU",
271
+ "type": "text",
272
+ "source": "odoo",
273
+ "default": false,
274
+ "description": "The product this customer spent the most on in the last 12 months."
275
+ },
276
+ {
277
+ "key": "days_since",
278
+ "label": "Days since order",
279
+ "type": "int",
280
+ "source": "odoo",
281
+ "default": false,
282
+ "description": "Days since the last confirmed order."
283
+ },
284
+ {
285
+ "key": "typical_gap_days",
286
+ "label": "Typical gap days",
287
+ "type": "int",
288
+ "source": "odoo",
289
+ "default": false,
290
+ "description": "Days this customer usually goes between orders, from their own history."
291
+ },
292
+ {
293
+ "key": "notes",
294
+ "label": "Notes",
295
+ "type": "text",
296
+ "source": "overlay",
297
+ "default": false,
298
+ "description": "Your notes on this customer. Saved in this app only, visible only to you."
299
+ }
300
+ ],
301
+ "_product_comment": "ADDITIVE, wave 15 C-TOPIC. The PRODUCT table's field contract. Kept as a SEPARATE top-level key rather than restructuring `fields` into {customer_data, product_data}: both existing readers (aios_grid._load_fields, aios-web/api/main.py) index doc['fields'] directly, and reshaping that mid-wave would break the embed for a cosmetic gain. The keyed shape can arrive when both readers move in ONE commit; until then this is the product half and `fields` is the customer half.",
302
+ "_product_removed_buy_now": "OWNER, 2026-08-03: 'Buy signal' (key buy_now, a select of Buy now / OK) is NO LONGER A PRESET FIELD. It never earned one: it is a formula over two columns that are both still right here, and the platform has a formula field type for exactly that. THE FORMULA, which reproduces the retired column row for row (modules/product_data.validate proves the equivalence, and goes red if it ever stops holding): IF({lead_days} > 0, IF({dos} < {lead_days}, \"Buy now\", \"OK\"), \"\") . Every branch matches the old server rule, including the blanks - the formula engine refuses a comparison against a blank rather than coercing it to 0, so a SKU with no days-of-supply or no lead time comes out empty, which is 'we do not know' and not 'you are fine'. NOTE the column is still COMPUTED in product_data.pool(): it ships nowhere (rows_from_pool projects strictly through this contract, so no Field means no cell on the wire) and exists only as validate()'s oracle. A formula field is PER-USER, so nothing shared may filter on it - the Buy list view filters on dos/lead_days directly (_seed_wave17).",
303
+ "product_data": {
304
+ "identity": "pid",
305
+ "business_key": "code",
306
+ "fields": [
307
+ {
308
+ "key": "code",
309
+ "label": "SKU",
310
+ "type": "text",
311
+ "source": "odoo",
312
+ "pinned": true,
313
+ "default": true,
314
+ "description": "The SKU code — the product's real business key. `pid` is a stable CRC32 of it because the grid keys on an integer."
315
+ },
316
+ {
317
+ "key": "product",
318
+ "label": "Product",
319
+ "type": "text",
320
+ "source": "odoo",
321
+ "default": true,
322
+ "description": "Product name as it appears in Odoo."
323
+ },
324
+ {
325
+ "key": "category",
326
+ "label": "Category",
327
+ "type": "select",
328
+ "source": "odoo",
329
+ "default": true,
330
+ "description": "Product category; '(uncategorized)' when Odoo carries none."
331
+ },
332
+ {
333
+ "key": "supplier",
334
+ "label": "Supplier",
335
+ "type": "text",
336
+ "source": "overlay",
337
+ "default": true,
338
+ "description": "Who makes it. Editable here and shared with everyone in the workspace; seeded from the inventory mastersheet.",
339
+ "shared": true
340
+ },
341
+ {
342
+ "key": "origin_country",
343
+ "label": "Country",
344
+ "type": "text",
345
+ "source": "overlay",
346
+ "default": false,
347
+ "description": "Country of origin. Editable here and shared with everyone; seeded from the inventory mastersheet.",
348
+ "shared": true
349
+ },
350
+ {
351
+ "key": "lead_days",
352
+ "label": "Lead time (days)",
353
+ "type": "int",
354
+ "source": "overlay",
355
+ "default": true,
356
+ "description": "Order-to-arrival days for this supplier. Drives the buy signal. Editable and shared with everyone.",
357
+ "shared": true
358
+ },
359
+ {
360
+ "key": "first_cost",
361
+ "label": "First cost",
362
+ "type": "currency",
363
+ "source": "overlay",
364
+ "default": false,
365
+ "description": "Quoted unit cost at origin, before freight and duty. Editable and shared with everyone.",
366
+ "shared": true
367
+ },
368
+ {
369
+ "key": "price_fisch",
370
+ "label": "Fisch price",
371
+ "type": "currency",
372
+ "source": "odoo",
373
+ "description": "Fisch pricelist price for this SKU. Blank when that list prices it nowhere."
374
+ },
375
+ {
376
+ "key": "price_royal_1",
377
+ "label": "Royal 1 price",
378
+ "type": "currency",
379
+ "source": "odoo",
380
+ "description": "Royal 1 pricelist price for this SKU. Blank when that list prices it nowhere."
381
+ },
382
+ {
383
+ "key": "price_royal_2",
384
+ "label": "Royal 2 price",
385
+ "type": "currency",
386
+ "source": "odoo",
387
+ "description": "Royal 2 pricelist price for this SKU. Blank when that list prices it nowhere."
388
+ },
389
+ {
390
+ "key": "tier_prices",
391
+ "label": "Tier prices",
392
+ "type": "json",
393
+ "source": "odoo",
394
+ "description": "Every live pricelist that prices this SKU today, as [{pricelist, unit_price}] at qty 1. Blank when no list prices it. This is the honest SET; the three Fisch/Royal columns beside it are the DECLARED subset and cannot show a price on a list the contract does not name."
395
+ },
396
+ {
397
+ "key": "units",
398
+ "label": "Units",
399
+ "type": "json",
400
+ "source": "odoo",
401
+ "description": "The units of measure this SKU is really sold in, as [{name, qty}] where qty is in the product's own unit. Blank means it is sold in ONE unit, not that data is missing - only 1,150 of 5,873 active SKUs (19.6%) carry a unit tier."
402
+ },
403
+ {
404
+ "key": "rev_ytd",
405
+ "label": "Revenue YTD",
406
+ "type": "currency",
407
+ "source": "odoo",
408
+ "default": true,
409
+ "description": "Year-to-date revenue for this SKU, BU-scoped when the caller is."
410
+ },
411
+ {
412
+ "key": "rev_ly",
413
+ "label": "Revenue LY",
414
+ "type": "currency",
415
+ "source": "odoo",
416
+ "description": "Same period last year — seasonal wholesale compares like for like."
417
+ },
418
+ {
419
+ "key": "yoy_pct",
420
+ "label": "YoY %",
421
+ "type": "pct",
422
+ "source": "odoo",
423
+ "description": "Year-over-year change; null when last year was zero (a ratio to zero is not a number)."
424
+ },
425
+ {
426
+ "key": "qty_ytd",
427
+ "label": "Units YTD",
428
+ "type": "int",
429
+ "source": "odoo",
430
+ "description": "Units sold year to date."
431
+ },
432
+ {
433
+ "key": "orders_ytd",
434
+ "label": "Orders YTD",
435
+ "type": "int",
436
+ "source": "odoo",
437
+ "description": "Distinct orders containing this SKU, year to date."
438
+ },
439
+ {
440
+ "key": "on_hand",
441
+ "label": "On hand",
442
+ "type": "int",
443
+ "source": "odoo",
444
+ "description": "Units in stock. CONSOLIDATED — one physical warehouse, not brand-tagged, so this column is ABSENT for a BU-scoped caller rather than silently company-wide."
445
+ },
446
+ {
447
+ "key": "unit_cost",
448
+ "label": "Unit cost",
449
+ "type": "currency",
450
+ "source": "odoo",
451
+ "description": "Inventory unit cost. Consolidated; absent for a BU-scoped caller."
452
+ },
453
+ {
454
+ "key": "inv_value",
455
+ "label": "Stock value",
456
+ "type": "currency",
457
+ "source": "odoo",
458
+ "description": "On-hand value at cost. Consolidated; absent for a BU-scoped caller."
459
+ },
460
+ {
461
+ "key": "qty_ltm",
462
+ "label": "Units LTM",
463
+ "type": "int",
464
+ "source": "odoo",
465
+ "description": "Units sold in the last twelve months. Consolidated; absent for a BU-scoped caller."
466
+ },
467
+ {
468
+ "key": "incoming",
469
+ "label": "Inbound units",
470
+ "type": "int",
471
+ "source": "odoo",
472
+ "description": "Units already on order and not yet received, from Odoo's own incoming quantity on the product. In the product's STOCK unit of measure, the same unit as On hand, so the two can be added. Days of supply and both cover gap columns all count this as stock (owner, 2026-08-19)."
473
+ },
474
+ {
475
+ "key": "dos",
476
+ "label": "Days of supply",
477
+ "type": "int",
478
+ "source": "odoo",
479
+ "description": "Days of supply at the trailing twelve month rate, counting ON HAND PLUS INBOUND units as stock; null means it never sells through. Same stock figure as the cover gap columns, so the two cannot disagree about how much you have. They still divide by different rates: this one is the trailing average, the cover gap uses the forward 8 month forecast, so a seasonal SKU reads differently in each. Consolidated; absent for a BU-scoped caller."
480
+ },
481
+ {
482
+ "key": "cover_gap_d",
483
+ "label": "Cover gap (days)",
484
+ "type": "int",
485
+ "source": "odoo",
486
+ "default": false,
487
+ "description": "Days of cover minus supplier lead time. Negative means it runs out before a reorder lands. It counts INBOUND units as stock, exactly as Days of supply now does. The one thing it does differently: the burn rate is the FORWARD 8 month forecast rather than the trailing twelve month average, so a seasonal SKU reads differently here."
488
+ },
489
+ {
490
+ "key": "cover_gap_units",
491
+ "label": "Cover gap (units)",
492
+ "type": "int",
493
+ "source": "odoo",
494
+ "default": false,
495
+ "description": "The recommended reorder quantity: units of forecast demand over the lead time that on-hand plus inbound does not cover. POSITIVE means buy this many; negative is surplus units; blank means we cannot say, because the SKU has no lead time on file or no forecast demand. Rounded AWAY from zero, so a real shortfall never rounds down to nothing."
496
+ },
497
+ {
498
+ "key": "demand_fwd",
499
+ "label": "Forecast units (8 mo)",
500
+ "type": "int",
501
+ "source": "odoo",
502
+ "description": "Units this SKU is expected to sell over the next 8 months, read as the units it actually sold in the same 8 calendar months one year ago. Seasonal on purpose: a flat annual average spreads Valentine's, Mother's Day and Christmas evenly across the year and understates the months buyers actually order for. A SKU too new to appear in that window falls back to its trailing twelve month rate scaled to 8 months. This is the burn rate both cover gap columns divide by, shown so the reorder quantity can be checked."
503
+ },
504
+ {
505
+ "key": "stock_bucket",
506
+ "label": "Stock status",
507
+ "type": "select",
508
+ "source": "odoo",
509
+ "description": "Dead / excess / healthy bucket, from the same days of supply figure beside it, so it counts inbound units too. One exception on purpose: 'Out of stock' still keys on the REAL shelf, because that is a present tense fact somebody can walk into the warehouse and check. A row can honestly read 'Out of stock' with a days of supply beside it when the replenishment is on the water; Inbound units is why. Consolidated; absent for a BU-scoped caller."
510
+ },
511
+ {
512
+ "key": "discontinued",
513
+ "label": "Discontinued",
514
+ "type": "select",
515
+ "source": "odoo",
516
+ "options": [
517
+ "Yes",
518
+ "No"
519
+ ],
520
+ "description": "Whether Odoo carries the Discontinued product tag on this SKU. Every row gets an explicit Yes or No rather than a blank, because a blank reads as an inactive condition in the filter engine and would silently WIDEN any view that filtered on it."
521
+ },
522
+ {
523
+ "key": "needs_pricing",
524
+ "label": "Needs pricing",
525
+ "type": "select",
526
+ "source": "overlay",
527
+ "default": false,
528
+ "options": [
529
+ "Yes"
530
+ ],
531
+ "shared": true,
532
+ "description": "Team-maintained. A SKU carries “Yes” when it appears on the NEEDS PRICING tab of the 2027 catalog workbook; no value means it is not on that list. Shared with everyone in the workspace — the whole point of R8 is that the TEAM's work lands in the system, not one importer's private column."
533
+ },
534
+ {
535
+ "key": "march_pricelist",
536
+ "label": "March pricelist",
537
+ "type": "select",
538
+ "source": "overlay",
539
+ "default": false,
540
+ "options": [
541
+ "Yes"
542
+ ],
543
+ "shared": true,
544
+ "description": "Team-maintained. A SKU carries “Yes” when it appears on the March Pricelist tab of the 2027 catalog workbook; no value means it is not on that list. Shared with everyone in the workspace — the whole point of R8 is that the TEAM's work lands in the system, not one importer's private column."
545
+ },
546
+ {
547
+ "key": "price_changes",
548
+ "label": "Price changes",
549
+ "type": "select",
550
+ "source": "overlay",
551
+ "default": false,
552
+ "options": [
553
+ "Yes"
554
+ ],
555
+ "shared": true,
556
+ "description": "Team-maintained. A SKU carries “Yes” when it appears on the Price Changes tab of the 2027 catalog workbook; no value means it is not on that list. Shared with everyone in the workspace — the whole point of R8 is that the TEAM's work lands in the system, not one importer's private column."
557
+ },
558
+ {
559
+ "key": "closeouts",
560
+ "label": "Closeouts",
561
+ "type": "select",
562
+ "source": "overlay",
563
+ "default": false,
564
+ "options": [
565
+ "Yes"
566
+ ],
567
+ "shared": true,
568
+ "description": "Team-maintained. A SKU carries “Yes” when it appears on the Closeouts tab of the 2027 catalog workbook; no value means it is not on that list. Shared with everyone in the workspace — the whole point of R8 is that the TEAM's work lands in the system, not one importer's private column."
569
+ },
570
+ {
571
+ "key": "notes",
572
+ "label": "Notes",
573
+ "type": "text",
574
+ "source": "overlay",
575
+ "default": false,
576
+ "shared": true,
577
+ "description": "Team-maintained. The 2027 workbook's own non-Odoo columns (Product Description, Packing) folded into one field. Shared with everyone in the workspace."
578
+ }
579
+ ]
580
+ }
581
+ }
platform/core/perm_scope.py CHANGED
The diff for this file is too large to render. See raw diff
 
platform/core/script_sandbox.py CHANGED
@@ -1,519 +1,519 @@
1
- """core/script_sandbox.py — WAVE 36 (R5 / R10, contract C1): running a tenant's OWN Python.
2
-
3
- Owner item 6: *"Add code script as an interface (database View) so a user can build whatever they
4
- want through the Agent chat interface."* Item 8: *"We need to really guardrail the reach of this
5
- script. So let's really grill this down."* R10 ruled it SERVER-SIDE PYTHON after the trade was
6
- stated, so this file is the guardrail, and one engine serves both items.
7
-
8
- ════════════════════════════════════════════════════════════════════════════════════════════════
9
- ⛔⛔ THE ONE PARAGRAPH TO READ BEFORE CHANGING ANYTHING HERE.
10
-
11
- In-process CPython cannot deliver two of this ticket's clauses. An AST allow-list plus a curated
12
- namespace stops import, file, network and environment access — but it **cannot cap memory and
13
- cannot interrupt a runaway loop**, because a `while True:` in the same interpreter is not a slow
14
- request, it is the tenant's ONE FastAPI process gone. So the script runs in a **SUBPROCESS**:
15
- `resource.setrlimit` for address space and CPU, a hard wall-clock kill from the parent, and the
16
- allow-list inside. Neither half is sufficient; both are load-bearing.
17
-
18
- ⭐ AND THE SUBPROCESS RECEIVES **ROWS, NEVER A STORE**. The parent calls C1's `scoped_table` under
19
- the CALLING user's record and serialises the result; the child imports nothing from this repo and
20
- holds no credential, no runtime and no store handle. Wiring W1 ("the sandbox has no second store
21
- path") is then true by CONSTRUCTION rather than by discipline, and it is checkable: the child
22
- reports its own `sys.modules`, and no `core.*` name may appear in it.
23
-
24
- ⛔ NEVER A BLACKLIST. Every rule below is an ALLOW-LIST — a set of node types, a set of attribute
25
- names, a dict of builtins. A blacklist of dangerous spellings is bypassable by construction, and
26
- the bypass is usually one string method away (`"{0.__class__}".format(x)` performs its attribute
27
- lookup inside `format`, so there is no `ast.Attribute` node to refuse).
28
- ════════════════════════════════════════════════════════════════════════════════════════════════
29
-
30
- The two layers, and they refuse DIFFERENT things on purpose:
31
-
32
- 1. `check_source()` — a pure function over source text. Refuses a construct the language offers
33
- and this sandbox does not: `import`, `class`, `with`, `async`, `yield`, `global`, and every
34
- attribute name outside `ALLOWED_ATTRS`.
35
- 2. `SANDBOX_BUILTINS` — the names that resolve at all. `__import__`, `open`, `eval`, `exec`,
36
- `compile`, `getattr`, `globals`, `vars` and `type` are simply absent, so a source that gets
37
- past layer 1 still finds nothing to call.
38
-
39
- ⚠ THAT DUPLICATION IS DELIBERATE AND IT CHANGES HOW THE GATE MUST BE WRITTEN. `import os` is
40
- refused twice, so a negative control that drops ONE layer sees the other refuse and reports
41
- green — the shape that already cost this wave one missed control in `routes_agent_harness`. So
42
- each layer is tested AT ITS OWN BOUNDARY: `check_source()` is called directly on source strings,
43
- and `run()` is driven end to end. An NC drops one entry from one frozenset and the matching
44
- boundary goes red.
45
- """
46
- import ast
47
- import json
48
- import os
49
- import subprocess
50
- import sys
51
- import tempfile
52
- import time
53
- from pathlib import Path
54
-
55
- #: Wall clock, enforced by the PARENT with a kill. The one cap that works on every platform.
56
- DEFAULT_TIMEOUT_S = 10.0
57
-
58
- #: Address space for the child (`RLIMIT_AS`). POSIX only — see `run()`'s `caps` report.
59
- DEFAULT_MEMORY_BYTES = 512 * 1024 * 1024
60
-
61
- #: CPU seconds for the child (`RLIMIT_CPU`). POSIX only. Deliberately above the wall clock: the
62
- #: wall-clock kill is the primary control and this is the backstop for a child that stops being
63
- #: reachable. A CPU limit BELOW the timeout would make every slow script look like a CPU refusal.
64
- DEFAULT_CPU_SECONDS = 15
65
-
66
- #: What the script may print, in bytes. `print` is a curated builtin writing to a capped buffer,
67
- #: and the child's real stdout goes to DEVNULL — so a script cannot fill a pipe, and anything
68
- #: that escaped far enough to write to fd 1 has nowhere for it to land.
69
- MAX_STDOUT_BYTES = 64 * 1024
70
-
71
- #: The serialised ROW payload handed to the child. ⛔ A REFUSAL, NEVER A TRUNCATION (standing rule
72
- #: 1): a short answer from a data tool is a wrong answer that looks right. Over this, `run()`
73
- #: returns a named limit carrying its cause and a recommendation.
74
- MAX_PAYLOAD_BYTES = 32 * 1024 * 1024
75
-
76
- #: The emitted spec. A render spec is a description of a picture; one larger than this is data
77
- #: pretending to be a description.
78
- MAX_SPEC_BYTES = 2 * 1024 * 1024
79
-
80
- MAX_SOURCE_BYTES = 128 * 1024
81
-
82
-
83
- # ══════════════════════════════════════════════════════ LAYER 1 — the AST allow-list ═══════════
84
- #: Every `ast` node class a script may contain. ⛔ THE ABSENCES ARE THE POLICY: `Import` /
85
- #: `ImportFrom` (no module reaches the script), `ClassDef` (a class body is a namespace with its
86
- #: own scoping rules and buys a data script nothing), `With` (a context manager is `__enter__`
87
- #: by another spelling), `Global` / `Nonlocal` (rebinding the sandbox's own names), and every
88
- #: `Async*` / `Await` / `Yield` form (this engine is synchronous; a coroutine that is never
89
- #: awaited is a silent no-op that looks like a working script).
90
- ALLOWED_NODES = frozenset("""
91
- Module Expr Assign AugAssign AnnAssign NamedExpr Return Pass Break Continue Delete Assert Raise
92
- If For While Try TryStar ExceptHandler FunctionDef Lambda arguments arg keyword
93
- BoolOp BinOp UnaryOp IfExp Dict Set List Tuple Starred Subscript Slice Compare Call Attribute Name
94
- Constant JoinedStr FormattedValue ListComp SetComp DictComp GeneratorExp comprehension
95
- Load Store Del
96
- And Or Not Invert UAdd USub
97
- Add Sub Mult Div FloorDiv Mod Pow LShift RShift BitOr BitXor BitAnd MatMult
98
- Eq NotEq Lt LtE Gt GtE Is IsNot In NotIn
99
- """.split())
100
-
101
- #: Every attribute name a script may READ or CALL. ⛔⛔ THIS IS THE LOAD-BEARING SET, and it is
102
- #: an allow-list of NAMES rather than a refusal of dunders, because the interesting escapes are
103
- #: ordinary-looking: `f.__globals__` on any function reaches the runner's own module namespace,
104
- #: `e.__traceback__.tb_frame.f_globals` reaches it from an exception handler, and `().__class__`
105
- #: reaches `object.__subclasses__`. None of those names is here, and neither is any name this
106
- #: sandbox has not been asked for.
107
- #: ⚠ `format` IS ABSENT DELIBERATELY. `"{0.__class__}".format(x)` performs the attribute lookup
108
- #: INSIDE `str.format`, where no `ast.Attribute` node exists for layer 1 to see. f-strings are
109
- #: fine — `f"{x.__class__}"` compiles to a real `Attribute` node and is refused.
110
- ALLOWED_ATTRS = frozenset("""
111
- append extend insert pop remove clear sort reverse copy count index
112
- keys values items get setdefault update
113
- add discard union intersection difference issubset issuperset
114
- join split rsplit splitlines strip lstrip rstrip lower upper title capitalize casefold
115
- replace startswith endswith find rfind zfill ljust rjust center partition removeprefix removesuffix
116
- isdigit isalpha isalnum isspace isupper islower isnumeric
117
- real imag numerator denominator
118
- """.split())
119
-
120
-
121
- class Refused(Exception):
122
- """A named refusal: `code` for a caller to branch on, `message` for a person to read."""
123
-
124
- def __init__(self, code, message):
125
- self.code, self.message = code, message
126
- super().__init__(f"{code}: {message}")
127
-
128
-
129
- def _attr_ok(name):
130
- """An attribute name passes only if it is on the list AND is not private.
131
-
132
- ⚠ THE SECOND TEST IS NOT A BLACKLIST — it narrows an allow-list that already excludes every
133
- private name. It is here so that adding a name to `ALLOWED_ATTRS` cannot open a dunder by
134
- accident, which is the one edit a future reader is most likely to make in a hurry.
135
- """
136
- return name in ALLOWED_ATTRS and not name.startswith("_")
137
-
138
-
139
- def check_source(source):
140
- """LAYER 1. Return a `Refused` for source this sandbox will not run, or `None`.
141
-
142
- ⭐ PURE, AND THAT IS WHAT MAKES IT TESTABLE AT ITS OWN BOUNDARY. It reads no file, spawns no
143
- process and touches no store, so a gate can hand it a hundred hostile strings for free and an
144
- NC can drop one entry from one frozenset and watch exactly this function change its answer.
145
- """
146
- text = str(source or "")
147
- if len(text.encode("utf-8", "replace")) > MAX_SOURCE_BYTES:
148
- return Refused("source_too_long",
149
- f"a script view is at most {MAX_SOURCE_BYTES // 1024} KB of source")
150
- try:
151
- tree = ast.parse(text)
152
- except SyntaxError as exc:
153
- return Refused("syntax", f"line {exc.lineno or 0}: {exc.msg}")
154
-
155
- for node in ast.walk(tree):
156
- kind = type(node).__name__
157
- if kind not in ALLOWED_NODES:
158
- return Refused("refused_construct",
159
- f"line {getattr(node, 'lineno', 0)}: this sandbox does not run "
160
- f"{_english(kind)}")
161
- if isinstance(node, ast.Attribute) and not _attr_ok(node.attr):
162
- return Refused("refused_attribute",
163
- f"line {getattr(node, 'lineno', 0)}: the attribute "
164
- f"'{node.attr}' is not available inside a script view")
165
- # ⛔ A NAME may not be private either. `_` prefixed names are the runner's own, and a
166
- # script that could bind one could shadow the machinery it runs on top of.
167
- if isinstance(node, ast.Name) and node.id.startswith("_"):
168
- return Refused("reserved_name",
169
- f"line {getattr(node, 'lineno', 0)}: names starting with an "
170
- f"underscore are reserved by the sandbox")
171
- if isinstance(node, (ast.FunctionDef, ast.arg, ast.ExceptHandler)) and str(
172
- getattr(node, "name", None) or getattr(node, "arg", "") or "").startswith("_"):
173
- return Refused("reserved_name",
174
- f"line {getattr(node, 'lineno', 0)}: names starting with an "
175
- f"underscore are reserved by the sandbox")
176
- if isinstance(node, ast.keyword) and str(node.arg or "").startswith("_"):
177
- return Refused("reserved_name",
178
- f"line {getattr(node, 'lineno', 0)}: keyword arguments starting with "
179
- f"an underscore are reserved by the sandbox")
180
- return None
181
-
182
-
183
- _ENGLISH = {
184
- "Import": "an import", "ImportFrom": "an import", "ClassDef": "a class definition",
185
- "With": "a with block", "AsyncWith": "a with block", "AsyncFor": "an async loop",
186
- "AsyncFunctionDef": "an async function", "Await": "await", "Yield": "yield",
187
- "YieldFrom": "yield from", "Global": "a global statement", "Nonlocal": "a nonlocal statement",
188
- "Match": "a match statement",
189
- }
190
-
191
-
192
- def _english(kind):
193
- return _ENGLISH.get(kind, f"a {kind} expression")
194
-
195
-
196
- # ══════════════════════════════════════════ LAYER 2 — the namespace, and the child program ═════
197
- #: The builtins a script may reach, BY NAME. Everything else is a `NameError` in the child.
198
- #: ⛔ THE ABSENCES, again, are the policy: `__import__` `open` `eval` `exec` `compile` `input`
199
- #: `getattr` `setattr` `delattr` `globals` `locals` `vars` `dir` `type` `super` `object` `help`
200
- #: `exit` `breakpoint` `memoryview` `id`. Several are harmless on their own; each one is a step
201
- #: on a published escape, and none has ever been asked for by a script that shapes rows.
202
- #: ⚠ THE EXCEPTION CLASSES ARE HERE BECAUSE `try:` IS, and a `try` block whose `except` clause
203
- #: cannot name what it catches is a construct that reads as supported and is not. They are safe
204
- #: for the same reason everything else is: `Exception.__subclasses__` needs an attribute this
205
- #: sandbox does not allow, so a class object in the namespace is a leaf, not a doorway.
206
- SANDBOX_BUILTIN_NAMES = (
207
- "abs all any bool bytes callable chr dict divmod enumerate filter float frozenset hash hex "
208
- "int isinstance issubclass iter len list map max min next oct ord pow range repr reversed "
209
- "round set slice sorted str sum tuple zip True False None "
210
- "Exception ValueError TypeError KeyError IndexError ZeroDivisionError ArithmeticError "
211
- "AttributeError StopIteration OverflowError"
212
- ).split()
213
-
214
- #: The literal program the child runs. It is TEXT rather than a module because the child must
215
- #: import nothing from this repo: a module would be found on `sys.path` and would drag `core`
216
- #: with it, which is exactly the second store path W1 forbids.
217
- #: ⚠ Every name in here is underscore-prefixed and layer 1 refuses a script from binding one, so
218
- #: the runner's own machinery cannot be shadowed by the source it executes.
219
- _RUNNER = r'''
220
- import json as _json, os as _os, sys as _sys
221
-
222
- _pay = _json.loads(open(_sys.argv[1], "r", encoding="utf-8").read())
223
- _out = {"ok": False, "code": "not_run", "message": "the script did not run",
224
- "stdout": "", "spec": None, "caps": {"wallClock": True, "memory": False, "cpu": False}}
225
-
226
- # ── the caps this platform can actually apply, reported either way (standing rule 1) ──────────
227
- try:
228
- import resource as _res
229
- _mem = int(_pay["memoryBytes"])
230
- _res.setrlimit(_res.RLIMIT_AS, (_mem, _mem))
231
- _out["caps"]["memory"] = True
232
- _cpu = int(_pay["cpuSeconds"])
233
- _res.setrlimit(_res.RLIMIT_CPU, (_cpu, _cpu))
234
- _out["caps"]["cpu"] = True
235
- except Exception:
236
- # `resource` is POSIX only. The wall-clock kill in the parent still applies, and `caps` says
237
- # which of the three held, never a silent partial.
238
- pass
239
-
240
- _printed = []
241
- _spent = [0]
242
- _LIMIT = int(_pay["maxStdout"])
243
-
244
-
245
- def _print(*_a, **_k):
246
- _text = (_k.get("sep") or " ").join(str(_x) for _x in _a) + (_k.get("end") or "\n")
247
- _room = _LIMIT - _spent[0]
248
- if _room > 0:
249
- _printed.append(_text[:_room])
250
- _spent[0] += len(_text)
251
-
252
-
253
- class _Refusal(Exception):
254
- """The SANDBOX refusing, as distinct from the SCRIPT failing.
255
-
256
- Without its own class these arrive as `ValueError`, indistinguishable from a `ValueError` the
257
- script raised itself, and the answer then says "refused" about an ordinary bug in the tenant's
258
- own code. Two different facts, two different codes.
259
- """
260
-
261
-
262
- _emitted = []
263
-
264
-
265
- def _emit(_spec):
266
- if not isinstance(_spec, dict):
267
- raise _Refusal("emit() takes a view spec, which is a dictionary")
268
- if _emitted:
269
- raise _Refusal("emit() was already called; a script view emits exactly one view")
270
- _emitted.append(_spec)
271
-
272
-
273
- _rows = _pay["rows"]
274
- _fields = _pay["fields"]
275
- _bound = _pay["table"]
276
-
277
-
278
- def _scoped_table(_table=None):
279
- if _table is not None and str(_table) != _bound:
280
- raise _Refusal(
281
- "this script view is bound to the database '" + _bound + "' and asked for '"
282
- + str(_table) + "'. A script view reads its own database only")
283
- return [dict(_r) for _r in _rows]
284
-
285
-
286
- def _scoped_fields():
287
- return [dict(_f) for _f in _fields]
288
-
289
-
290
- _ns = {"__builtins__": {_n: __builtins__[_n] if isinstance(__builtins__, dict)
291
- else getattr(__builtins__, _n)
292
- for _n in _pay["builtins"]}}
293
- _ns["__builtins__"]["print"] = _print
294
- _ns["print"] = _print
295
- _ns["emit"] = _emit
296
- _ns["scoped_table"] = _scoped_table
297
- _ns["scoped_fields"] = _scoped_fields
298
- _ns["table"] = _bound
299
-
300
- try:
301
- exec(compile(_pay["source"], "<script view>", "exec"), _ns)
302
- if not _emitted:
303
- _out.update(ok=False, code="no_view",
304
- message="the script finished without calling emit(spec)")
305
- else:
306
- _out.update(ok=True, code="", message="", spec=_emitted[0])
307
- except _Refusal as _e:
308
- _out.update(ok=False, code="refused", message=str(_e)[:400])
309
- except MemoryError:
310
- _out.update(ok=False, code="memory",
311
- message="the script used more memory than a script view is allowed")
312
- except NameError as _e:
313
- _out.update(ok=False, code="refused_name",
314
- message=str(_e)[:200] + ". A script view may use only the names the sandbox "
315
- "provides")
316
- except BaseException as _e:
317
- _out.update(ok=False, code="error",
318
- message=type(_e).__name__ + ": " + str(_e)[:400])
319
-
320
- _out["stdout"] = "".join(_printed)
321
- _out["truncated"] = _spent[0] > _LIMIT
322
- # ⭐ THE PROBE: what this child actually had. The PARENT strips it unless it was asked for, so a
323
- # production run never carries it and a gate can still prove that no `core.*` module and no
324
- # secret-shaped environment key was ever inside this process.
325
- # ⚠ SNAPSHOTTED AFTER `exec`, and `os` is imported at the TOP so this list does not depend on
326
- # dict-literal evaluation order. The first draft called `__import__("os")` inside this very
327
- # expression, so whether `os` appeared depended on which value Python built first: a probe whose
328
- # contents move with an unrelated edit is a probe a gate cannot assert against.
329
- _out["probe"] = {"modules": sorted(_sys.modules), "env": sorted(_os.environ)}
330
- open(_sys.argv[2], "w", encoding="utf-8").write(_json.dumps(_out, default=str))
331
- '''
332
-
333
-
334
- def _child_env():
335
- """The child's WHOLE environment. An allow-list of two keys, and neither is a credential.
336
-
337
- ⛔ NOT `os.environ.copy()` MINUS SOMETHING. A subtractive environment ships every key nobody
338
- thought to name: `HF_TOKEN`, `ODOO_PASSWORD`, `ANTHROPIC_API_KEY` and whatever the next
339
- connector adds. The three names below are here because Python will not start on Windows
340
- without them; on Linux this returns `{}` and the child runs with no environment at all.
341
- """
342
- env = {}
343
- for name in ("SystemRoot", "SYSTEMROOT", "WINDIR"):
344
- if os.environ.get(name):
345
- env[name] = os.environ[name]
346
- return env
347
-
348
-
349
- def run(source, rows, fields, table_key, *, timeout_s=DEFAULT_TIMEOUT_S,
350
- memory_bytes=DEFAULT_MEMORY_BYTES, cpu_seconds=DEFAULT_CPU_SECONDS, probe=False):
351
- """Run ONE script over rows that are ALREADY scoped. Returns C3's envelope plus `caps`.
352
-
353
- {ok, code, message, spec, stdout, truncated, ms, caps: {wallClock, memory, cpu}}
354
-
355
- ⛔ THIS FUNCTION NEVER TOUCHES A STORE, AND THAT IS THE POINT: it takes rows. `run_view()`
356
- below is the door that fetches them through C1; keeping the two apart is what lets a gate
357
- drive the sandbox with no tenant, no runtime and no credential anywhere in the process.
358
-
359
- ⚠ `caps` IS PART OF THE ANSWER, NOT DEBUG OUTPUT. On Windows `resource` does not exist, so
360
- the memory and CPU limits are NOT applied and this says so. A caller that reports `ok:true`
361
- without reading `caps` is claiming an enforcement that did not happen (standing rule 1).
362
- """
363
- started = time.monotonic()
364
- refusal = check_source(source)
365
- if refusal is not None:
366
- return _refusal(refusal.code, refusal.message, started)
367
-
368
- payload = {"source": str(source or ""), "rows": rows, "fields": fields,
369
- "table": str(table_key or ""), "builtins": SANDBOX_BUILTIN_NAMES,
370
- "maxStdout": MAX_STDOUT_BYTES, "memoryBytes": int(memory_bytes),
371
- "cpuSeconds": int(cpu_seconds)}
372
- try:
373
- blob = json.dumps(payload, default=str)
374
- except (TypeError, ValueError) as exc:
375
- return _refusal("bad_rows", f"these rows cannot be handed to a script ({exc})", started)
376
- if len(blob.encode("utf-8", "replace")) > MAX_PAYLOAD_BYTES:
377
- # ⛔ REPORTED, NOT TRUNCATED (standing rule 1's second sentence): cause and recommendation,
378
- # in the words the owner asked for, rather than a quietly short answer.
379
- return _refusal(
380
- "payload_too_large",
381
- f"this database's rows are larger than the {MAX_PAYLOAD_BYTES // (1024 * 1024)} MB a "
382
- f"script view can be handed at once. Narrow the view with a filter, or raise the "
383
- f"sandbox payload limit for this deployment", started)
384
-
385
- with tempfile.TemporaryDirectory(prefix="aios-script-") as work:
386
- pay_path = Path(work) / "payload.json"
387
- res_path = Path(work) / "result.json"
388
- pay_path.write_text(blob, encoding="utf-8")
389
- # `-I` isolates the interpreter (no PYTHON* env, no user site), `-S` skips site-packages,
390
- # and the program arrives on STDIN so there is no file for anything to import it as.
391
- # ⛔ `-X utf8` AND AN EXPLICIT `encoding` ARE NOT TIDINESS. Without them this pipe is
392
- # encoded with the parent's locale codec, which on this Windows box is cp1252: the runner
393
- # text below cannot be represented in it and `subprocess.run` died with
394
- # `UnicodeEncodeError` before the child ever started. A sandbox whose behaviour depends on
395
- # the operator's locale is a sandbox with two behaviours. `-I` implies `-E`, so
396
- # `PYTHONUTF8` in the environment could not have carried this, it has to be a flag.
397
- argv = [sys.executable, "-I", "-S", "-X", "utf8", "-", str(pay_path), str(res_path)]
398
- try:
399
- done = subprocess.run(
400
- argv, input=_RUNNER, text=True, encoding="utf-8", errors="replace",
401
- cwd=work, env=_child_env(),
402
- stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, timeout=timeout_s)
403
- except subprocess.TimeoutExpired:
404
- return _refusal("timeout",
405
- f"the script ran longer than {timeout_s:g} seconds and was stopped",
406
- started)
407
- except OSError as exc:
408
- return _refusal("no_sandbox",
409
- f"a script view could not be started on this deployment ({exc})",
410
- started)
411
-
412
- if not res_path.is_file():
413
- # The child died without writing an answer: an rlimit signal, an OOM kill, or a crash.
414
- # ⚠ NAMED BY ITS RETURN CODE rather than reported as a generic failure — a memory kill
415
- # and a bug in this file must not read identically to an operator.
416
- return _refusal(*_died(done), started)
417
- try:
418
- out = json.loads(res_path.read_text(encoding="utf-8"))
419
- except (OSError, ValueError) as exc:
420
- return _refusal("unreadable", f"the script's answer could not be read ({exc})",
421
- started)
422
-
423
- out["ms"] = int((time.monotonic() - started) * 1000)
424
- if out.get("ok"):
425
- spec_error = _check_spec(out.get("spec"))
426
- if spec_error:
427
- out.update(ok=False, code="bad_spec", message=spec_error, spec=None)
428
- if not probe:
429
- out.pop("probe", None)
430
- return out
431
-
432
-
433
- def _died(done):
434
- """`(code, message)` for a child that produced no answer."""
435
- rc = done.returncode
436
- tail = " ".join((done.stderr or "").split())[-300:]
437
- if rc in (-9, 137):
438
- return "memory", "the script was stopped for using too much memory"
439
- if rc in (-24, 152):
440
- return "timeout", "the script used more processor time than a script view is allowed"
441
- return "crashed", f"the script view engine stopped without an answer{': ' + tail if tail else ''}"
442
-
443
-
444
- def _refusal(code, message, started):
445
- return {"ok": False, "code": code, "message": message, "spec": None, "stdout": "",
446
- "truncated": False, "ms": int((time.monotonic() - started) * 1000),
447
- "caps": {"wallClock": True, "memory": False, "cpu": False}}
448
-
449
-
450
- def _check_spec(spec):
451
- """C3: a spec is a DESCRIPTION the client draws. Never HTML, never a script, never a URL.
452
-
453
- ⛔ THE CHECK IS ON THE KEYS, NOT ON THE STRING CONTENTS. Scanning values for `<script>` is a
454
- blacklist and would pass `<SCR` + `IPT>`; refusing a spec that carries an `html`, `script`,
455
- `src` or `onclick` key refuses the SHAPE that would let a renderer be talked into executing
456
- something. The vocabulary of legal `kind`s is the ROUTE's business (W36-T37) — this is the
457
- floor every caller gets whether or not the route above it remembers.
458
- """
459
- if not isinstance(spec, dict):
460
- return "the script emitted something that is not a view spec"
461
- try:
462
- blob = json.dumps(spec)
463
- except (TypeError, ValueError):
464
- return "the emitted view spec is not something the client can be sent"
465
- if len(blob.encode("utf-8", "replace")) > MAX_SPEC_BYTES:
466
- return (f"the emitted view spec is over {MAX_SPEC_BYTES // (1024 * 1024)} MB. A view spec "
467
- f"describes a picture; it is not where the rows go")
468
- banned = {"html", "innerhtml", "script", "src", "srcdoc", "href", "style", "onclick", "onload"}
469
- found = sorted(k for k in _keys_of(spec) if str(k).lower() in banned)
470
- if found:
471
- return (f"a view spec may not carry {', '.join(found)}. The client DRAWS a spec, so a "
472
- f"markup or URL key would be a script by another name")
473
- return None
474
-
475
-
476
- def _keys_of(value, depth=0):
477
- """Every key anywhere in a nested spec. Bounded, so a deep structure cannot spin this."""
478
- if depth > 12:
479
- return
480
- if isinstance(value, dict):
481
- for key, sub in value.items():
482
- yield key
483
- yield from _keys_of(sub, depth + 1)
484
- elif isinstance(value, (list, tuple)):
485
- for sub in value:
486
- yield from _keys_of(sub, depth + 1)
487
-
488
-
489
- # ══════════════════════════════════════════════ THE DOOR — C1 is the ONLY way to a row ═════════
490
- def run_view(user, table_key, source, st=None, **kw):
491
- """Fetch through C1 under `user`'s scope, then run the script over what came back (R5).
492
-
493
- ⭐⭐ THE FETCH HAPPENS IN THE PARENT AND ONLY ROWS CROSS INTO THE CHILD. That is wiring W1
494
- made structural: the child has no runtime to ask, no store handle to open and no credential
495
- to use, so "a script cannot read what its caller cannot read" is not a rule anybody has to
496
- keep — there is no second path for it to be broken through.
497
-
498
- ⛔ C1'S THREE EXCEPTIONS ARE ANSWERED, NEVER SWALLOWED. `UnknownTable`, `Denied` and
499
- `Unresolvable` mean three different things to a person; collapsing them into "no rows" is the
500
- silent-empty answer C1 was written to make impossible. `Unresolvable.as_limit()` is handed
501
- through in the words it was raised with — standing rule 1's second sentence, verbatim.
502
- """
503
- import core.perm_scope as perm_scope
504
-
505
- key = str(table_key or "")
506
- try:
507
- rows = perm_scope.scoped_table(user, key, st=st)
508
- fields = perm_scope.scoped_fields(user, key, st=st)
509
- except perm_scope.UnknownTable as exc:
510
- return _refusal("unknown_table", str(exc) or f"there is no database '{key}'",
511
- time.monotonic())
512
- except perm_scope.Denied as exc:
513
- return _refusal("denied", str(exc) or "this account may not read that database",
514
- time.monotonic())
515
- except perm_scope.Unresolvable as exc:
516
- out = _refusal("unresolvable", str(exc), time.monotonic())
517
- out["limit"] = exc.as_limit()
518
- return out
519
- return run(source, rows, fields, key, **kw)
 
1
+ """core/script_sandbox.py — WAVE 36 (R5 / R10, contract C1): running a tenant's OWN Python.
2
+
3
+ Owner item 6: *"Add code script as an interface (database View) so a user can build whatever they
4
+ want through the Agent chat interface."* Item 8: *"We need to really guardrail the reach of this
5
+ script. So let's really grill this down."* R10 ruled it SERVER-SIDE PYTHON after the trade was
6
+ stated, so this file is the guardrail, and one engine serves both items.
7
+
8
+ ════════════════════════════════════════════════════════════════════════════════════════════════
9
+ ⛔⛔ THE ONE PARAGRAPH TO READ BEFORE CHANGING ANYTHING HERE.
10
+
11
+ In-process CPython cannot deliver two of this ticket's clauses. An AST allow-list plus a curated
12
+ namespace stops import, file, network and environment access — but it **cannot cap memory and
13
+ cannot interrupt a runaway loop**, because a `while True:` in the same interpreter is not a slow
14
+ request, it is the tenant's ONE FastAPI process gone. So the script runs in a **SUBPROCESS**:
15
+ `resource.setrlimit` for address space and CPU, a hard wall-clock kill from the parent, and the
16
+ allow-list inside. Neither half is sufficient; both are load-bearing.
17
+
18
+ ⭐ AND THE SUBPROCESS RECEIVES **ROWS, NEVER A STORE**. The parent calls C1's `scoped_table` under
19
+ the CALLING user's record and serialises the result; the child imports nothing from this repo and
20
+ holds no credential, no runtime and no store handle. Wiring W1 ("the sandbox has no second store
21
+ path") is then true by CONSTRUCTION rather than by discipline, and it is checkable: the child
22
+ reports its own `sys.modules`, and no `core.*` name may appear in it.
23
+
24
+ ⛔ NEVER A BLACKLIST. Every rule below is an ALLOW-LIST — a set of node types, a set of attribute
25
+ names, a dict of builtins. A blacklist of dangerous spellings is bypassable by construction, and
26
+ the bypass is usually one string method away (`"{0.__class__}".format(x)` performs its attribute
27
+ lookup inside `format`, so there is no `ast.Attribute` node to refuse).
28
+ ════════════════════════════════════════════════════════════════════════════════════════════════
29
+
30
+ The two layers, and they refuse DIFFERENT things on purpose:
31
+
32
+ 1. `check_source()` — a pure function over source text. Refuses a construct the language offers
33
+ and this sandbox does not: `import`, `class`, `with`, `async`, `yield`, `global`, and every
34
+ attribute name outside `ALLOWED_ATTRS`.
35
+ 2. `SANDBOX_BUILTINS` — the names that resolve at all. `__import__`, `open`, `eval`, `exec`,
36
+ `compile`, `getattr`, `globals`, `vars` and `type` are simply absent, so a source that gets
37
+ past layer 1 still finds nothing to call.
38
+
39
+ ⚠ THAT DUPLICATION IS DELIBERATE AND IT CHANGES HOW THE GATE MUST BE WRITTEN. `import os` is
40
+ refused twice, so a negative control that drops ONE layer sees the other refuse and reports
41
+ green — the shape that already cost this wave one missed control in `routes_agent_harness`. So
42
+ each layer is tested AT ITS OWN BOUNDARY: `check_source()` is called directly on source strings,
43
+ and `run()` is driven end to end. An NC drops one entry from one frozenset and the matching
44
+ boundary goes red.
45
+ """
46
+ import ast
47
+ import json
48
+ import os
49
+ import subprocess
50
+ import sys
51
+ import tempfile
52
+ import time
53
+ from pathlib import Path
54
+
55
+ #: Wall clock, enforced by the PARENT with a kill. The one cap that works on every platform.
56
+ DEFAULT_TIMEOUT_S = 10.0
57
+
58
+ #: Address space for the child (`RLIMIT_AS`). POSIX only — see `run()`'s `caps` report.
59
+ DEFAULT_MEMORY_BYTES = 512 * 1024 * 1024
60
+
61
+ #: CPU seconds for the child (`RLIMIT_CPU`). POSIX only. Deliberately above the wall clock: the
62
+ #: wall-clock kill is the primary control and this is the backstop for a child that stops being
63
+ #: reachable. A CPU limit BELOW the timeout would make every slow script look like a CPU refusal.
64
+ DEFAULT_CPU_SECONDS = 15
65
+
66
+ #: What the script may print, in bytes. `print` is a curated builtin writing to a capped buffer,
67
+ #: and the child's real stdout goes to DEVNULL — so a script cannot fill a pipe, and anything
68
+ #: that escaped far enough to write to fd 1 has nowhere for it to land.
69
+ MAX_STDOUT_BYTES = 64 * 1024
70
+
71
+ #: The serialised ROW payload handed to the child. ⛔ A REFUSAL, NEVER A TRUNCATION (standing rule
72
+ #: 1): a short answer from a data tool is a wrong answer that looks right. Over this, `run()`
73
+ #: returns a named limit carrying its cause and a recommendation.
74
+ MAX_PAYLOAD_BYTES = 32 * 1024 * 1024
75
+
76
+ #: The emitted spec. A render spec is a description of a picture; one larger than this is data
77
+ #: pretending to be a description.
78
+ MAX_SPEC_BYTES = 2 * 1024 * 1024
79
+
80
+ MAX_SOURCE_BYTES = 128 * 1024
81
+
82
+
83
+ # ══════════════════════════════════════════════════════ LAYER 1 — the AST allow-list ═══════════
84
+ #: Every `ast` node class a script may contain. ⛔ THE ABSENCES ARE THE POLICY: `Import` /
85
+ #: `ImportFrom` (no module reaches the script), `ClassDef` (a class body is a namespace with its
86
+ #: own scoping rules and buys a data script nothing), `With` (a context manager is `__enter__`
87
+ #: by another spelling), `Global` / `Nonlocal` (rebinding the sandbox's own names), and every
88
+ #: `Async*` / `Await` / `Yield` form (this engine is synchronous; a coroutine that is never
89
+ #: awaited is a silent no-op that looks like a working script).
90
+ ALLOWED_NODES = frozenset("""
91
+ Module Expr Assign AugAssign AnnAssign NamedExpr Return Pass Break Continue Delete Assert Raise
92
+ If For While Try TryStar ExceptHandler FunctionDef Lambda arguments arg keyword
93
+ BoolOp BinOp UnaryOp IfExp Dict Set List Tuple Starred Subscript Slice Compare Call Attribute Name
94
+ Constant JoinedStr FormattedValue ListComp SetComp DictComp GeneratorExp comprehension
95
+ Load Store Del
96
+ And Or Not Invert UAdd USub
97
+ Add Sub Mult Div FloorDiv Mod Pow LShift RShift BitOr BitXor BitAnd MatMult
98
+ Eq NotEq Lt LtE Gt GtE Is IsNot In NotIn
99
+ """.split())
100
+
101
+ #: Every attribute name a script may READ or CALL. ⛔⛔ THIS IS THE LOAD-BEARING SET, and it is
102
+ #: an allow-list of NAMES rather than a refusal of dunders, because the interesting escapes are
103
+ #: ordinary-looking: `f.__globals__` on any function reaches the runner's own module namespace,
104
+ #: `e.__traceback__.tb_frame.f_globals` reaches it from an exception handler, and `().__class__`
105
+ #: reaches `object.__subclasses__`. None of those names is here, and neither is any name this
106
+ #: sandbox has not been asked for.
107
+ #: ⚠ `format` IS ABSENT DELIBERATELY. `"{0.__class__}".format(x)` performs the attribute lookup
108
+ #: INSIDE `str.format`, where no `ast.Attribute` node exists for layer 1 to see. f-strings are
109
+ #: fine — `f"{x.__class__}"` compiles to a real `Attribute` node and is refused.
110
+ ALLOWED_ATTRS = frozenset("""
111
+ append extend insert pop remove clear sort reverse copy count index
112
+ keys values items get setdefault update
113
+ add discard union intersection difference issubset issuperset
114
+ join split rsplit splitlines strip lstrip rstrip lower upper title capitalize casefold
115
+ replace startswith endswith find rfind zfill ljust rjust center partition removeprefix removesuffix
116
+ isdigit isalpha isalnum isspace isupper islower isnumeric
117
+ real imag numerator denominator
118
+ """.split())
119
+
120
+
121
+ class Refused(Exception):
122
+ """A named refusal: `code` for a caller to branch on, `message` for a person to read."""
123
+
124
+ def __init__(self, code, message):
125
+ self.code, self.message = code, message
126
+ super().__init__(f"{code}: {message}")
127
+
128
+
129
+ def _attr_ok(name):
130
+ """An attribute name passes only if it is on the list AND is not private.
131
+
132
+ ⚠ THE SECOND TEST IS NOT A BLACKLIST — it narrows an allow-list that already excludes every
133
+ private name. It is here so that adding a name to `ALLOWED_ATTRS` cannot open a dunder by
134
+ accident, which is the one edit a future reader is most likely to make in a hurry.
135
+ """
136
+ return name in ALLOWED_ATTRS and not name.startswith("_")
137
+
138
+
139
+ def check_source(source):
140
+ """LAYER 1. Return a `Refused` for source this sandbox will not run, or `None`.
141
+
142
+ ⭐ PURE, AND THAT IS WHAT MAKES IT TESTABLE AT ITS OWN BOUNDARY. It reads no file, spawns no
143
+ process and touches no store, so a gate can hand it a hundred hostile strings for free and an
144
+ NC can drop one entry from one frozenset and watch exactly this function change its answer.
145
+ """
146
+ text = str(source or "")
147
+ if len(text.encode("utf-8", "replace")) > MAX_SOURCE_BYTES:
148
+ return Refused("source_too_long",
149
+ f"a script view is at most {MAX_SOURCE_BYTES // 1024} KB of source")
150
+ try:
151
+ tree = ast.parse(text)
152
+ except SyntaxError as exc:
153
+ return Refused("syntax", f"line {exc.lineno or 0}: {exc.msg}")
154
+
155
+ for node in ast.walk(tree):
156
+ kind = type(node).__name__
157
+ if kind not in ALLOWED_NODES:
158
+ return Refused("refused_construct",
159
+ f"line {getattr(node, 'lineno', 0)}: this sandbox does not run "
160
+ f"{_english(kind)}")
161
+ if isinstance(node, ast.Attribute) and not _attr_ok(node.attr):
162
+ return Refused("refused_attribute",
163
+ f"line {getattr(node, 'lineno', 0)}: the attribute "
164
+ f"'{node.attr}' is not available inside a script view")
165
+ # ⛔ A NAME may not be private either. `_` prefixed names are the runner's own, and a
166
+ # script that could bind one could shadow the machinery it runs on top of.
167
+ if isinstance(node, ast.Name) and node.id.startswith("_"):
168
+ return Refused("reserved_name",
169
+ f"line {getattr(node, 'lineno', 0)}: names starting with an "
170
+ f"underscore are reserved by the sandbox")
171
+ if isinstance(node, (ast.FunctionDef, ast.arg, ast.ExceptHandler)) and str(
172
+ getattr(node, "name", None) or getattr(node, "arg", "") or "").startswith("_"):
173
+ return Refused("reserved_name",
174
+ f"line {getattr(node, 'lineno', 0)}: names starting with an "
175
+ f"underscore are reserved by the sandbox")
176
+ if isinstance(node, ast.keyword) and str(node.arg or "").startswith("_"):
177
+ return Refused("reserved_name",
178
+ f"line {getattr(node, 'lineno', 0)}: keyword arguments starting with "
179
+ f"an underscore are reserved by the sandbox")
180
+ return None
181
+
182
+
183
+ _ENGLISH = {
184
+ "Import": "an import", "ImportFrom": "an import", "ClassDef": "a class definition",
185
+ "With": "a with block", "AsyncWith": "a with block", "AsyncFor": "an async loop",
186
+ "AsyncFunctionDef": "an async function", "Await": "await", "Yield": "yield",
187
+ "YieldFrom": "yield from", "Global": "a global statement", "Nonlocal": "a nonlocal statement",
188
+ "Match": "a match statement",
189
+ }
190
+
191
+
192
+ def _english(kind):
193
+ return _ENGLISH.get(kind, f"a {kind} expression")
194
+
195
+
196
+ # ══════════════════════════════════════════ LAYER 2 — the namespace, and the child program ═════
197
+ #: The builtins a script may reach, BY NAME. Everything else is a `NameError` in the child.
198
+ #: ⛔ THE ABSENCES, again, are the policy: `__import__` `open` `eval` `exec` `compile` `input`
199
+ #: `getattr` `setattr` `delattr` `globals` `locals` `vars` `dir` `type` `super` `object` `help`
200
+ #: `exit` `breakpoint` `memoryview` `id`. Several are harmless on their own; each one is a step
201
+ #: on a published escape, and none has ever been asked for by a script that shapes rows.
202
+ #: ⚠ THE EXCEPTION CLASSES ARE HERE BECAUSE `try:` IS, and a `try` block whose `except` clause
203
+ #: cannot name what it catches is a construct that reads as supported and is not. They are safe
204
+ #: for the same reason everything else is: `Exception.__subclasses__` needs an attribute this
205
+ #: sandbox does not allow, so a class object in the namespace is a leaf, not a doorway.
206
+ SANDBOX_BUILTIN_NAMES = (
207
+ "abs all any bool bytes callable chr dict divmod enumerate filter float frozenset hash hex "
208
+ "int isinstance issubclass iter len list map max min next oct ord pow range repr reversed "
209
+ "round set slice sorted str sum tuple zip True False None "
210
+ "Exception ValueError TypeError KeyError IndexError ZeroDivisionError ArithmeticError "
211
+ "AttributeError StopIteration OverflowError"
212
+ ).split()
213
+
214
+ #: The literal program the child runs. It is TEXT rather than a module because the child must
215
+ #: import nothing from this repo: a module would be found on `sys.path` and would drag `core`
216
+ #: with it, which is exactly the second store path W1 forbids.
217
+ #: ⚠ Every name in here is underscore-prefixed and layer 1 refuses a script from binding one, so
218
+ #: the runner's own machinery cannot be shadowed by the source it executes.
219
+ _RUNNER = r'''
220
+ import json as _json, os as _os, sys as _sys
221
+
222
+ _pay = _json.loads(open(_sys.argv[1], "r", encoding="utf-8").read())
223
+ _out = {"ok": False, "code": "not_run", "message": "the script did not run",
224
+ "stdout": "", "spec": None, "caps": {"wallClock": True, "memory": False, "cpu": False}}
225
+
226
+ # ── the caps this platform can actually apply, reported either way (standing rule 1) ──────────
227
+ try:
228
+ import resource as _res
229
+ _mem = int(_pay["memoryBytes"])
230
+ _res.setrlimit(_res.RLIMIT_AS, (_mem, _mem))
231
+ _out["caps"]["memory"] = True
232
+ _cpu = int(_pay["cpuSeconds"])
233
+ _res.setrlimit(_res.RLIMIT_CPU, (_cpu, _cpu))
234
+ _out["caps"]["cpu"] = True
235
+ except Exception:
236
+ # `resource` is POSIX only. The wall-clock kill in the parent still applies, and `caps` says
237
+ # which of the three held, never a silent partial.
238
+ pass
239
+
240
+ _printed = []
241
+ _spent = [0]
242
+ _LIMIT = int(_pay["maxStdout"])
243
+
244
+
245
+ def _print(*_a, **_k):
246
+ _text = (_k.get("sep") or " ").join(str(_x) for _x in _a) + (_k.get("end") or "\n")
247
+ _room = _LIMIT - _spent[0]
248
+ if _room > 0:
249
+ _printed.append(_text[:_room])
250
+ _spent[0] += len(_text)
251
+
252
+
253
+ class _Refusal(Exception):
254
+ """The SANDBOX refusing, as distinct from the SCRIPT failing.
255
+
256
+ Without its own class these arrive as `ValueError`, indistinguishable from a `ValueError` the
257
+ script raised itself, and the answer then says "refused" about an ordinary bug in the tenant's
258
+ own code. Two different facts, two different codes.
259
+ """
260
+
261
+
262
+ _emitted = []
263
+
264
+
265
+ def _emit(_spec):
266
+ if not isinstance(_spec, dict):
267
+ raise _Refusal("emit() takes a view spec, which is a dictionary")
268
+ if _emitted:
269
+ raise _Refusal("emit() was already called; a script view emits exactly one view")
270
+ _emitted.append(_spec)
271
+
272
+
273
+ _rows = _pay["rows"]
274
+ _fields = _pay["fields"]
275
+ _bound = _pay["table"]
276
+
277
+
278
+ def _scoped_table(_table=None):
279
+ if _table is not None and str(_table) != _bound:
280
+ raise _Refusal(
281
+ "this script view is bound to the database '" + _bound + "' and asked for '"
282
+ + str(_table) + "'. A script view reads its own database only")
283
+ return [dict(_r) for _r in _rows]
284
+
285
+
286
+ def _scoped_fields():
287
+ return [dict(_f) for _f in _fields]
288
+
289
+
290
+ _ns = {"__builtins__": {_n: __builtins__[_n] if isinstance(__builtins__, dict)
291
+ else getattr(__builtins__, _n)
292
+ for _n in _pay["builtins"]}}
293
+ _ns["__builtins__"]["print"] = _print
294
+ _ns["print"] = _print
295
+ _ns["emit"] = _emit
296
+ _ns["scoped_table"] = _scoped_table
297
+ _ns["scoped_fields"] = _scoped_fields
298
+ _ns["table"] = _bound
299
+
300
+ try:
301
+ exec(compile(_pay["source"], "<script view>", "exec"), _ns)
302
+ if not _emitted:
303
+ _out.update(ok=False, code="no_view",
304
+ message="the script finished without calling emit(spec)")
305
+ else:
306
+ _out.update(ok=True, code="", message="", spec=_emitted[0])
307
+ except _Refusal as _e:
308
+ _out.update(ok=False, code="refused", message=str(_e)[:400])
309
+ except MemoryError:
310
+ _out.update(ok=False, code="memory",
311
+ message="the script used more memory than a script view is allowed")
312
+ except NameError as _e:
313
+ _out.update(ok=False, code="refused_name",
314
+ message=str(_e)[:200] + ". A script view may use only the names the sandbox "
315
+ "provides")
316
+ except BaseException as _e:
317
+ _out.update(ok=False, code="error",
318
+ message=type(_e).__name__ + ": " + str(_e)[:400])
319
+
320
+ _out["stdout"] = "".join(_printed)
321
+ _out["truncated"] = _spent[0] > _LIMIT
322
+ # ⭐ THE PROBE: what this child actually had. The PARENT strips it unless it was asked for, so a
323
+ # production run never carries it and a gate can still prove that no `core.*` module and no
324
+ # secret-shaped environment key was ever inside this process.
325
+ # ⚠ SNAPSHOTTED AFTER `exec`, and `os` is imported at the TOP so this list does not depend on
326
+ # dict-literal evaluation order. The first draft called `__import__("os")` inside this very
327
+ # expression, so whether `os` appeared depended on which value Python built first: a probe whose
328
+ # contents move with an unrelated edit is a probe a gate cannot assert against.
329
+ _out["probe"] = {"modules": sorted(_sys.modules), "env": sorted(_os.environ)}
330
+ open(_sys.argv[2], "w", encoding="utf-8").write(_json.dumps(_out, default=str))
331
+ '''
332
+
333
+
334
+ def _child_env():
335
+ """The child's WHOLE environment. An allow-list of two keys, and neither is a credential.
336
+
337
+ ⛔ NOT `os.environ.copy()` MINUS SOMETHING. A subtractive environment ships every key nobody
338
+ thought to name: `HF_TOKEN`, `ODOO_PASSWORD`, `ANTHROPIC_API_KEY` and whatever the next
339
+ connector adds. The three names below are here because Python will not start on Windows
340
+ without them; on Linux this returns `{}` and the child runs with no environment at all.
341
+ """
342
+ env = {}
343
+ for name in ("SystemRoot", "SYSTEMROOT", "WINDIR"):
344
+ if os.environ.get(name):
345
+ env[name] = os.environ[name]
346
+ return env
347
+
348
+
349
+ def run(source, rows, fields, table_key, *, timeout_s=DEFAULT_TIMEOUT_S,
350
+ memory_bytes=DEFAULT_MEMORY_BYTES, cpu_seconds=DEFAULT_CPU_SECONDS, probe=False):
351
+ """Run ONE script over rows that are ALREADY scoped. Returns C3's envelope plus `caps`.
352
+
353
+ {ok, code, message, spec, stdout, truncated, ms, caps: {wallClock, memory, cpu}}
354
+
355
+ ⛔ THIS FUNCTION NEVER TOUCHES A STORE, AND THAT IS THE POINT: it takes rows. `run_view()`
356
+ below is the door that fetches them through C1; keeping the two apart is what lets a gate
357
+ drive the sandbox with no tenant, no runtime and no credential anywhere in the process.
358
+
359
+ ⚠ `caps` IS PART OF THE ANSWER, NOT DEBUG OUTPUT. On Windows `resource` does not exist, so
360
+ the memory and CPU limits are NOT applied and this says so. A caller that reports `ok:true`
361
+ without reading `caps` is claiming an enforcement that did not happen (standing rule 1).
362
+ """
363
+ started = time.monotonic()
364
+ refusal = check_source(source)
365
+ if refusal is not None:
366
+ return _refusal(refusal.code, refusal.message, started)
367
+
368
+ payload = {"source": str(source or ""), "rows": rows, "fields": fields,
369
+ "table": str(table_key or ""), "builtins": SANDBOX_BUILTIN_NAMES,
370
+ "maxStdout": MAX_STDOUT_BYTES, "memoryBytes": int(memory_bytes),
371
+ "cpuSeconds": int(cpu_seconds)}
372
+ try:
373
+ blob = json.dumps(payload, default=str)
374
+ except (TypeError, ValueError) as exc:
375
+ return _refusal("bad_rows", f"these rows cannot be handed to a script ({exc})", started)
376
+ if len(blob.encode("utf-8", "replace")) > MAX_PAYLOAD_BYTES:
377
+ # ⛔ REPORTED, NOT TRUNCATED (standing rule 1's second sentence): cause and recommendation,
378
+ # in the words the owner asked for, rather than a quietly short answer.
379
+ return _refusal(
380
+ "payload_too_large",
381
+ f"this database's rows are larger than the {MAX_PAYLOAD_BYTES // (1024 * 1024)} MB a "
382
+ f"script view can be handed at once. Narrow the view with a filter, or raise the "
383
+ f"sandbox payload limit for this deployment", started)
384
+
385
+ with tempfile.TemporaryDirectory(prefix="aios-script-") as work:
386
+ pay_path = Path(work) / "payload.json"
387
+ res_path = Path(work) / "result.json"
388
+ pay_path.write_text(blob, encoding="utf-8")
389
+ # `-I` isolates the interpreter (no PYTHON* env, no user site), `-S` skips site-packages,
390
+ # and the program arrives on STDIN so there is no file for anything to import it as.
391
+ # ⛔ `-X utf8` AND AN EXPLICIT `encoding` ARE NOT TIDINESS. Without them this pipe is
392
+ # encoded with the parent's locale codec, which on this Windows box is cp1252: the runner
393
+ # text below cannot be represented in it and `subprocess.run` died with
394
+ # `UnicodeEncodeError` before the child ever started. A sandbox whose behaviour depends on
395
+ # the operator's locale is a sandbox with two behaviours. `-I` implies `-E`, so
396
+ # `PYTHONUTF8` in the environment could not have carried this, it has to be a flag.
397
+ argv = [sys.executable, "-I", "-S", "-X", "utf8", "-", str(pay_path), str(res_path)]
398
+ try:
399
+ done = subprocess.run(
400
+ argv, input=_RUNNER, text=True, encoding="utf-8", errors="replace",
401
+ cwd=work, env=_child_env(),
402
+ stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, timeout=timeout_s)
403
+ except subprocess.TimeoutExpired:
404
+ return _refusal("timeout",
405
+ f"the script ran longer than {timeout_s:g} seconds and was stopped",
406
+ started)
407
+ except OSError as exc:
408
+ return _refusal("no_sandbox",
409
+ f"a script view could not be started on this deployment ({exc})",
410
+ started)
411
+
412
+ if not res_path.is_file():
413
+ # The child died without writing an answer: an rlimit signal, an OOM kill, or a crash.
414
+ # ⚠ NAMED BY ITS RETURN CODE rather than reported as a generic failure — a memory kill
415
+ # and a bug in this file must not read identically to an operator.
416
+ return _refusal(*_died(done), started)
417
+ try:
418
+ out = json.loads(res_path.read_text(encoding="utf-8"))
419
+ except (OSError, ValueError) as exc:
420
+ return _refusal("unreadable", f"the script's answer could not be read ({exc})",
421
+ started)
422
+
423
+ out["ms"] = int((time.monotonic() - started) * 1000)
424
+ if out.get("ok"):
425
+ spec_error = _check_spec(out.get("spec"))
426
+ if spec_error:
427
+ out.update(ok=False, code="bad_spec", message=spec_error, spec=None)
428
+ if not probe:
429
+ out.pop("probe", None)
430
+ return out
431
+
432
+
433
+ def _died(done):
434
+ """`(code, message)` for a child that produced no answer."""
435
+ rc = done.returncode
436
+ tail = " ".join((done.stderr or "").split())[-300:]
437
+ if rc in (-9, 137):
438
+ return "memory", "the script was stopped for using too much memory"
439
+ if rc in (-24, 152):
440
+ return "timeout", "the script used more processor time than a script view is allowed"
441
+ return "crashed", f"the script view engine stopped without an answer{': ' + tail if tail else ''}"
442
+
443
+
444
+ def _refusal(code, message, started):
445
+ return {"ok": False, "code": code, "message": message, "spec": None, "stdout": "",
446
+ "truncated": False, "ms": int((time.monotonic() - started) * 1000),
447
+ "caps": {"wallClock": True, "memory": False, "cpu": False}}
448
+
449
+
450
+ def _check_spec(spec):
451
+ """C3: a spec is a DESCRIPTION the client draws. Never HTML, never a script, never a URL.
452
+
453
+ ⛔ THE CHECK IS ON THE KEYS, NOT ON THE STRING CONTENTS. Scanning values for `<script>` is a
454
+ blacklist and would pass `<SCR` + `IPT>`; refusing a spec that carries an `html`, `script`,
455
+ `src` or `onclick` key refuses the SHAPE that would let a renderer be talked into executing
456
+ something. The vocabulary of legal `kind`s is the ROUTE's business (W36-T37) — this is the
457
+ floor every caller gets whether or not the route above it remembers.
458
+ """
459
+ if not isinstance(spec, dict):
460
+ return "the script emitted something that is not a view spec"
461
+ try:
462
+ blob = json.dumps(spec)
463
+ except (TypeError, ValueError):
464
+ return "the emitted view spec is not something the client can be sent"
465
+ if len(blob.encode("utf-8", "replace")) > MAX_SPEC_BYTES:
466
+ return (f"the emitted view spec is over {MAX_SPEC_BYTES // (1024 * 1024)} MB. A view spec "
467
+ f"describes a picture; it is not where the rows go")
468
+ banned = {"html", "innerhtml", "script", "src", "srcdoc", "href", "style", "onclick", "onload"}
469
+ found = sorted(k for k in _keys_of(spec) if str(k).lower() in banned)
470
+ if found:
471
+ return (f"a view spec may not carry {', '.join(found)}. The client DRAWS a spec, so a "
472
+ f"markup or URL key would be a script by another name")
473
+ return None
474
+
475
+
476
+ def _keys_of(value, depth=0):
477
+ """Every key anywhere in a nested spec. Bounded, so a deep structure cannot spin this."""
478
+ if depth > 12:
479
+ return
480
+ if isinstance(value, dict):
481
+ for key, sub in value.items():
482
+ yield key
483
+ yield from _keys_of(sub, depth + 1)
484
+ elif isinstance(value, (list, tuple)):
485
+ for sub in value:
486
+ yield from _keys_of(sub, depth + 1)
487
+
488
+
489
+ # ══════════════════════════════════════════════ THE DOOR — C1 is the ONLY way to a row ═════════
490
+ def run_view(user, table_key, source, st=None, **kw):
491
+ """Fetch through C1 under `user`'s scope, then run the script over what came back (R5).
492
+
493
+ ⭐⭐ THE FETCH HAPPENS IN THE PARENT AND ONLY ROWS CROSS INTO THE CHILD. That is wiring W1
494
+ made structural: the child has no runtime to ask, no store handle to open and no credential
495
+ to use, so "a script cannot read what its caller cannot read" is not a rule anybody has to
496
+ keep — there is no second path for it to be broken through.
497
+
498
+ ⛔ C1'S THREE EXCEPTIONS ARE ANSWERED, NEVER SWALLOWED. `UnknownTable`, `Denied` and
499
+ `Unresolvable` mean three different things to a person; collapsing them into "no rows" is the
500
+ silent-empty answer C1 was written to make impossible. `Unresolvable.as_limit()` is handed
501
+ through in the words it was raised with — standing rule 1's second sentence, verbatim.
502
+ """
503
+ import core.perm_scope as perm_scope
504
+
505
+ key = str(table_key or "")
506
+ try:
507
+ rows = perm_scope.scoped_table(user, key, st=st)
508
+ fields = perm_scope.scoped_fields(user, key, st=st)
509
+ except perm_scope.UnknownTable as exc:
510
+ return _refusal("unknown_table", str(exc) or f"there is no database '{key}'",
511
+ time.monotonic())
512
+ except perm_scope.Denied as exc:
513
+ return _refusal("denied", str(exc) or "this account may not read that database",
514
+ time.monotonic())
515
+ except perm_scope.Unresolvable as exc:
516
+ out = _refusal("unresolvable", str(exc), time.monotonic())
517
+ out["limit"] = exc.as_limit()
518
+ return out
519
+ return run(source, rows, fields, key, **kw)
platform/core/shares.py CHANGED
@@ -1,296 +1,296 @@
1
- """core/shares.py — ONE grant registry for every shareable object (wave 20, owner ruling R10).
2
-
3
- WHAT R10 ASKED FOR: folders and databases share with the **same two-role vocabulary views
4
- already use** (specific users or everyone; role = view | edit), plus one manage-access editor
5
- that can add or revoke people later, on any of the three.
6
-
7
- WHY A REGISTRY RATHER THAN A FIELD ON EACH OBJECT. A view already carries its own `permissions`
8
- (`core/table_store.py`) and that stays — moving it would rewrite every stored view for no gain.
9
- But a FOLDER is a value inside one user's workspace blob and a DATABASE is a `user_tables`
10
- definition; giving each its own grant field would put the same three-line permission decision in
11
- three files owned by two sessions, which is how the three drift. One registry, one predicate,
12
- three callers.
13
-
14
- shares.set_grants(kind, oid, entries, owner=…, st=…) # replaces the whole grant set
15
- shares.grants(kind, oid, st=…) # -> {'owner': str, 'entries': [...]}
16
- shares.role_for(kind, oid, user, is_admin=…, st=…) # -> 'owner'|'edit'|'view'|None
17
- shares.shared_with(user, kind=…, st=…) # -> [oid] this user was granted
18
-
19
- THE ROLE VOCABULARY IS TWO WORDS AND THE DEFAULT IS THE NARROW ONE. `view` = may open and read;
20
- `edit` = may also change the object's CONTENT. Neither ever means "may re-share": changing grants
21
- is the OWNER's (or an admin's), which is `table_store._may_administer`'s existing rule promoted to
22
- every kind. A collaborator who could rewrite grants could grant themselves sole ownership of
23
- somebody else's object, or quietly widen a users-scoped share to everyone.
24
-
25
- ⛔ AN UNREADABLE GRANT IS NO GRANT. Every path here fails closed — junk in the bucket, a missing
26
- owner, an unknown role string all resolve to None rather than to a default that opens something.
27
- [[aios-permissioning]]: no fail-open defaults, ever.
28
-
29
- ⚠ THE BUCKET IS TENANT-SCOPED THROUGH `st`, like every other product-data write. Passing the
30
- session's `TenantRuntime` is what keeps Nurilab's grants in Nurilab's store; the module default
31
- (`core.store`) is tenant #0 and exists for the same reason it does everywhere else — the ~28
32
- callers that predate multi-tenancy. (This is the D-5/D-16 residency shape, and this module does
33
- NOT repeat their mistake: `st` is threaded from the first line rather than retrofitted.)
34
- """
35
- import core.store as store
36
-
37
- #: The store key. One bucket per tenant holds every kind's grants, because "what am I shared on"
38
- #: is a question across kinds — the "Shared with me" folder (R10) is exactly that query, and
39
- #: three separate buckets would make it three reads that can disagree about what a user can see.
40
- SHARES_KEY = 'object_shares'
41
-
42
- #: The shareable kinds. A CLOSED vocabulary: an unknown kind raises rather than creating a new
43
- #: namespace by typo, which would silently grant nothing to nobody and read as "sharing is broken".
44
- #:
45
- #: ⭐⭐ W38-T16 — `field` IS THE FOURTH, AND NOTHING IN THIS FILE BRANCHES ON IT. Every function
46
- #: below treats `kind` as an opaque bucket key (`_check_kind / grants / set_grants / role_for /
47
- #: may_see / may_edit / may_administer / shared_with / drop_objects`), so the kind's whole cost
48
- #: here is this tuple member. That is the point of one registry: the new object's WALL is written
49
- #: once in `core.perm_scope`, and its DOORS once in `routes_shares.py` — never a fourth
50
- #: permission decision in a fourth file ([[one-evaluator-per-question]]).
51
- KINDS = ('view', 'folder', 'database', 'field')
52
-
53
- #: `*` is "everyone who can already open the surface". It is NOT "every account on the platform".
54
- #: Spelled as a single character so it can never collide with a username (usernames are lower-case
55
- #: and non-empty by `core/users.py`, and are checked against this explicitly below).
56
- #:
57
- #: ⛔⛔ **AND WHAT THAT MEANS DEPENDS ON THE KIND — THE LINE THIS NOTE USED TO CARRY WAS FALSE FOR
58
- #: ONE OF THE THREE** (W33-T30, `waves/wave32/sharing-audit.md` S-8). It read *"the module/table
59
- #: wall runs FIRST and this never widens past it"*, flatly, and the audit's own words for that are
60
- #: *"the third docstring in this audit describing a check that is not on the path"*. Corrected
61
- #: here rather than deleted, because the sentence is TRUE of two kinds and the difference is the
62
- #: whole point:
63
- #: * `kind='view'` / `kind='folder'` on a GOVERNED module (`customer_data`, `product_data`) —
64
- #: the sentence holds. `require_session` plus the topic's own gate run first, and the
65
- #: receiver's row scope and hidden-field closure are applied BEFORE any foreign view is
66
- #: merged, so a grant can only narrow-or-equal what that account could already reach.
67
- #: * `kind='database'` on a `ut_*` table — ⭐⭐ **THE SENTENCE HOLDS HERE TOO NOW, AND THAT IS
68
- #: W36-T21 / OWNER RULING R6 (audit S-8, CLOSED).** It did not until wave 36, and the reason
69
- #: is worth keeping: `routes_admin._clean_perms` 400'd any key outside
70
- #: `("customer_data", "product_data")`, so no row filter and no hidden field could even be
71
- #: DECLARED for a user table, `routes_tables.py` made zero `perm_scope` calls, and it passed
72
- #: `hidden_keys=frozenset()`. A `database` grant was therefore ALL-OR-NOTHING — every row,
73
- #: every column — and this registry was the only wall behind it.
74
- #:
75
- #: ⭐ WHAT CHANGED: `perm_scope.scoped_table` (contract C1) is the ONE door to any database's
76
- #: rows. `routes_tables` applies the permanent filter before `pids` is taken and the transitive
77
- #: hidden-field closure after `workspace_wire`, on EVERY `ut_*` read — the same code that walls
78
- #: `customer_data` — and a door that cannot apply them REFUSES rather than serving the lot. So a
79
- #: `database` grant is once again bounded by a second wall: it decides WHETHER an account reaches
80
- #: the database, and C1 decides WHICH rows and columns it then sees. An `*` grant still admits
81
- #: every account in the tenant, and each of them still only sees what their own wall allows.
82
- #:
83
- #: ⚠ `routes_shares.py`'s module docstring carries the OLD sentence at the other door and is in
84
- #: no wave-36 fence — the audit's own fix was "say so at both", so one of the two is now stale.
85
- #: Booked in `mailbox/C.md` (C-14) rather than edited across a fence
86
- #: [[two-gates-can-assert-opposite-things]].
87
- EVERYONE = '*'
88
-
89
- ROLES = ('view', 'edit')
90
-
91
- #: ⛔⛔ A FIELD's OBJECT ID IS TOPIC-QUALIFIED, AND THE SEPARATOR IS DECLARED HERE SO THERE IS ONE
92
- #: SPELLING OF IT. A bare field key is NOT unique: `notes` exists on a dozen databases, and a
93
- #: grant stored under it would admit a grantee to every `notes` column in the tenant at once —
94
- #: the widening direction, silently, forever. `table_key` is the qualifier because it is the same
95
- #: identifier `shared_overlay.bucket()` already keys the values by, so the grant and the data it
96
- #: governs are named by the same string ([[one-question-two-normalizers]]).
97
- #:
98
- #: ⚠ A `ut_*` key and a registry topic key both match `[a-z0-9_]+` and neither can contain `:`,
99
- #: so the split below is unambiguous in both directions.
100
- FIELD_OID_SEP = ':'
101
-
102
-
103
- def field_oid(table_key, field_key):
104
- """`"<table_key>:<field_key>"` — the share id of ONE column on ONE database."""
105
- table = str(table_key or '').strip()
106
- field = str(field_key or '').strip()
107
- if not table or not field:
108
- raise ValueError('shares.field_oid: a field share names BOTH a database and a column. '
109
- 'A bare field key repeats across tables and would grant all of them')
110
- return f'{table}{FIELD_OID_SEP}{field}'
111
-
112
-
113
- def split_field_oid(oid):
114
- """`(table_key, field_key)` or `(None, None)` for anything that is not a field oid.
115
-
116
- ⛔ FAIL-CLOSED ON JUNK, like every other read here: a caller that cannot learn WHICH database
117
- an id names must not fall back to "the one I happen to be looking at", which is how a grant
118
- on somebody else's column would be read as a grant on this one.
119
- """
120
- raw = str(oid or '')
121
- table, sep, field = raw.partition(FIELD_OID_SEP)
122
- if not sep or not table.strip() or not field.strip() or FIELD_OID_SEP in field:
123
- return (None, None)
124
- return (table.strip(), field.strip())
125
-
126
-
127
- def _st(st):
128
- return st if st is not None else store
129
-
130
-
131
- def _check_kind(kind):
132
- k = str(kind or '').strip().lower()
133
- if k not in KINDS:
134
- raise ValueError(f'{kind!r} is not a shareable kind. Use one of {", ".join(KINDS)}. '
135
- f'This door refuses to invent a namespace from a typo.')
136
- return k
137
-
138
-
139
- def _clean_entries(entries):
140
- """Normalise + REJECT junk, returning [{'user': str, 'role': 'view'|'edit'}].
141
-
142
- Silently dropping a malformed entry is right here and wrong elsewhere: the caller is a UI
143
- that just listed the people it is about to grant, so a rejected row must not abort the whole
144
- save — but an entry with an unknown ROLE must not be stored as something else's default
145
- either. Dropped, never coerced.
146
- """
147
- out, seen = [], set()
148
- for e in entries or ():
149
- if not isinstance(e, dict):
150
- continue
151
- user = str(e.get('user') or '').strip().lower()
152
- role = str(e.get('role') or '').strip().lower()
153
- if not user or role not in ROLES or user in seen:
154
- continue
155
- seen.add(user)
156
- out.append({'user': user, 'role': role})
157
- return out
158
-
159
-
160
- def grants(kind, oid, st=None):
161
- """`{'owner': str|None, 'entries': [{'user','role'}]}` — never raises on a junk bucket."""
162
- kind = _check_kind(kind)
163
- try:
164
- bucket = (_st(st).get(SHARES_KEY) or {}).get(kind) or {}
165
- rec = bucket.get(str(oid)) or {}
166
- except Exception:
167
- return {'owner': None, 'entries': []}
168
- if not isinstance(rec, dict):
169
- return {'owner': None, 'entries': []}
170
- return {'owner': (str(rec.get('owner')).strip().lower() if rec.get('owner') else None),
171
- 'entries': _clean_entries(rec.get('entries'))}
172
-
173
-
174
- def set_grants(kind, oid, entries, owner=None, st=None):
175
- """REPLACE the grant set for one object. Returns the stored record.
176
-
177
- ⚠ REPLACE, NOT MERGE, and that is the contract the UI needs: revoking is expressed by an
178
- entry's ABSENCE. A merge-only API cannot remove anybody without a second verb, and the
179
- manage-access editor R10 asks for is exactly "here is the list now".
180
- """
181
- kind = _check_kind(kind)
182
- oid = str(oid)
183
- clean = _clean_entries(entries)
184
- owner_l = str(owner).strip().lower() if owner else None
185
-
186
- def _apply(data):
187
- by_kind = dict(data.get(kind) or {})
188
- prior = by_kind.get(oid) if isinstance(by_kind.get(oid), dict) else {}
189
- # The owner is STICKY: set once, and a later save that omits it must not orphan the
190
- # object. An ownerless grant record cannot answer "who may re-share this", so every
191
- # administer check would fail closed and the object would become unmanageable.
192
- keep_owner = owner_l or (str(prior.get('owner')).strip().lower()
193
- if prior.get('owner') else None)
194
- if not clean and not keep_owner:
195
- by_kind.pop(oid, None) # fully un-shared and unowned: leave no empty husk
196
- else:
197
- by_kind[oid] = {'owner': keep_owner, 'entries': clean}
198
- data[kind] = by_kind
199
- return data
200
-
201
- _st(st).update(SHARES_KEY, _apply, flush='async')
202
- return grants(kind, oid, st=st)
203
-
204
-
205
- def role_for(kind, oid, user, is_admin=False, st=None):
206
- """`'owner'` | `'edit'` | `'view'` | `None` — the caller's effective role, fail-closed.
207
-
208
- An ADMIN reads as `'owner'`: an admin who could not administer an object could not
209
- administer the tenant either, which is `table_store._may_administer`'s existing rule and is
210
- kept identical here so the two cannot disagree about the same view.
211
- """
212
- user = str(user or '').strip().lower()
213
- if not user:
214
- return None
215
- rec = grants(kind, oid, st=st)
216
- if is_admin or (rec['owner'] and rec['owner'] == user):
217
- return 'owner'
218
- best = None
219
- for e in rec['entries']:
220
- if e['user'] == user or e['user'] == EVERYONE:
221
- # The STRONGER of the two wins when both a personal and an everyone grant exist:
222
- # naming somebody explicitly is how you RAISE them above the room, so an
223
- # everyone-view + alice-edit pair must leave alice editing.
224
- if e['role'] == 'edit':
225
- return 'edit'
226
- best = best or 'view'
227
- return best
228
-
229
-
230
- def may_see(kind, oid, user, is_admin=False, st=None):
231
- return role_for(kind, oid, user, is_admin=is_admin, st=st) is not None
232
-
233
-
234
- def may_edit(kind, oid, user, is_admin=False, st=None):
235
- return role_for(kind, oid, user, is_admin=is_admin, st=st) in ('owner', 'edit')
236
-
237
-
238
- def may_administer(kind, oid, user, is_admin=False, st=None):
239
- """Only the owner or an admin may change grants or delete. See the module note on why this
240
- is deliberately narrower than `may_edit`."""
241
- return role_for(kind, oid, user, is_admin=is_admin, st=st) == 'owner'
242
-
243
-
244
- def shared_with(user, kind=None, st=None):
245
- """Every object id this user has been granted (excluding what they own).
246
-
247
- This is the "Shared with me" query (R10). It EXCLUDES owned objects deliberately: a folder
248
- you made is not something shared *with* you, and listing it there would make the system
249
- folder a duplicate of the rail above it.
250
- """
251
- user = str(user or '').strip().lower()
252
- if not user:
253
- return {}
254
- try:
255
- data = _st(st).get(SHARES_KEY) or {}
256
- except Exception:
257
- return {}
258
- out = {}
259
- for k in ([_check_kind(kind)] if kind else KINDS):
260
- hits = []
261
- for oid, rec in (data.get(k) or {}).items():
262
- if not isinstance(rec, dict):
263
- continue
264
- owner = str(rec.get('owner') or '').strip().lower()
265
- if owner == user:
266
- continue
267
- for e in _clean_entries(rec.get('entries')):
268
- if e['user'] in (user, EVERYONE):
269
- hits.append(str(oid))
270
- break
271
- out[k] = sorted(hits)
272
- return out if kind is None else {_check_kind(kind): out[_check_kind(kind)]}
273
-
274
-
275
- def drop_objects(pairs, st=None):
276
- """Remove whole grant RECORDS, owner husk included — wave 21, item 6a (C3).
277
-
278
- A deleted object's grants must die with it: `shared_with` would otherwise serve ghost ids
279
- into every receiver's "Shared with me" forever, and the ghost would 404 on open. One
280
- transaction for the whole sweep — a table delete drops its database grant plus a view
281
- grant per view that lived in its bucket."""
282
- want = {}
283
- for kind, oid in pairs or ():
284
- want.setdefault(_check_kind(kind), set()).add(str(oid))
285
- if not want:
286
- return
287
-
288
- def _apply(data):
289
- for kind, oids in want.items():
290
- by_kind = data.get(kind)
291
- if isinstance(by_kind, dict):
292
- for oid in oids:
293
- by_kind.pop(oid, None)
294
- return data
295
-
296
- _st(st).update(SHARES_KEY, _apply, flush='async')
 
1
+ """core/shares.py — ONE grant registry for every shareable object (wave 20, owner ruling R10).
2
+
3
+ WHAT R10 ASKED FOR: folders and databases share with the **same two-role vocabulary views
4
+ already use** (specific users or everyone; role = view | edit), plus one manage-access editor
5
+ that can add or revoke people later, on any of the three.
6
+
7
+ WHY A REGISTRY RATHER THAN A FIELD ON EACH OBJECT. A view already carries its own `permissions`
8
+ (`core/table_store.py`) and that stays — moving it would rewrite every stored view for no gain.
9
+ But a FOLDER is a value inside one user's workspace blob and a DATABASE is a `user_tables`
10
+ definition; giving each its own grant field would put the same three-line permission decision in
11
+ three files owned by two sessions, which is how the three drift. One registry, one predicate,
12
+ three callers.
13
+
14
+ shares.set_grants(kind, oid, entries, owner=…, st=…) # replaces the whole grant set
15
+ shares.grants(kind, oid, st=…) # -> {'owner': str, 'entries': [...]}
16
+ shares.role_for(kind, oid, user, is_admin=…, st=…) # -> 'owner'|'edit'|'view'|None
17
+ shares.shared_with(user, kind=…, st=…) # -> [oid] this user was granted
18
+
19
+ THE ROLE VOCABULARY IS TWO WORDS AND THE DEFAULT IS THE NARROW ONE. `view` = may open and read;
20
+ `edit` = may also change the object's CONTENT. Neither ever means "may re-share": changing grants
21
+ is the OWNER's (or an admin's), which is `table_store._may_administer`'s existing rule promoted to
22
+ every kind. A collaborator who could rewrite grants could grant themselves sole ownership of
23
+ somebody else's object, or quietly widen a users-scoped share to everyone.
24
+
25
+ ⛔ AN UNREADABLE GRANT IS NO GRANT. Every path here fails closed — junk in the bucket, a missing
26
+ owner, an unknown role string all resolve to None rather than to a default that opens something.
27
+ [[aios-permissioning]]: no fail-open defaults, ever.
28
+
29
+ ⚠ THE BUCKET IS TENANT-SCOPED THROUGH `st`, like every other product-data write. Passing the
30
+ session's `TenantRuntime` is what keeps Nurilab's grants in Nurilab's store; the module default
31
+ (`core.store`) is tenant #0 and exists for the same reason it does everywhere else — the ~28
32
+ callers that predate multi-tenancy. (This is the D-5/D-16 residency shape, and this module does
33
+ NOT repeat their mistake: `st` is threaded from the first line rather than retrofitted.)
34
+ """
35
+ import core.store as store
36
+
37
+ #: The store key. One bucket per tenant holds every kind's grants, because "what am I shared on"
38
+ #: is a question across kinds — the "Shared with me" folder (R10) is exactly that query, and
39
+ #: three separate buckets would make it three reads that can disagree about what a user can see.
40
+ SHARES_KEY = 'object_shares'
41
+
42
+ #: The shareable kinds. A CLOSED vocabulary: an unknown kind raises rather than creating a new
43
+ #: namespace by typo, which would silently grant nothing to nobody and read as "sharing is broken".
44
+ #:
45
+ #: ⭐⭐ W38-T16 — `field` IS THE FOURTH, AND NOTHING IN THIS FILE BRANCHES ON IT. Every function
46
+ #: below treats `kind` as an opaque bucket key (`_check_kind / grants / set_grants / role_for /
47
+ #: may_see / may_edit / may_administer / shared_with / drop_objects`), so the kind's whole cost
48
+ #: here is this tuple member. That is the point of one registry: the new object's WALL is written
49
+ #: once in `core.perm_scope`, and its DOORS once in `routes_shares.py` — never a fourth
50
+ #: permission decision in a fourth file ([[one-evaluator-per-question]]).
51
+ KINDS = ('view', 'folder', 'database', 'field')
52
+
53
+ #: `*` is "everyone who can already open the surface". It is NOT "every account on the platform".
54
+ #: Spelled as a single character so it can never collide with a username (usernames are lower-case
55
+ #: and non-empty by `core/users.py`, and are checked against this explicitly below).
56
+ #:
57
+ #: ⛔⛔ **AND WHAT THAT MEANS DEPENDS ON THE KIND — THE LINE THIS NOTE USED TO CARRY WAS FALSE FOR
58
+ #: ONE OF THE THREE** (W33-T30, `waves/wave32/sharing-audit.md` S-8). It read *"the module/table
59
+ #: wall runs FIRST and this never widens past it"*, flatly, and the audit's own words for that are
60
+ #: *"the third docstring in this audit describing a check that is not on the path"*. Corrected
61
+ #: here rather than deleted, because the sentence is TRUE of two kinds and the difference is the
62
+ #: whole point:
63
+ #: * `kind='view'` / `kind='folder'` on a GOVERNED module (`customer_data`, `product_data`) —
64
+ #: the sentence holds. `require_session` plus the topic's own gate run first, and the
65
+ #: receiver's row scope and hidden-field closure are applied BEFORE any foreign view is
66
+ #: merged, so a grant can only narrow-or-equal what that account could already reach.
67
+ #: * `kind='database'` on a `ut_*` table — ⭐⭐ **THE SENTENCE HOLDS HERE TOO NOW, AND THAT IS
68
+ #: W36-T21 / OWNER RULING R6 (audit S-8, CLOSED).** It did not until wave 36, and the reason
69
+ #: is worth keeping: `routes_admin._clean_perms` 400'd any key outside
70
+ #: `("customer_data", "product_data")`, so no row filter and no hidden field could even be
71
+ #: DECLARED for a user table, `routes_tables.py` made zero `perm_scope` calls, and it passed
72
+ #: `hidden_keys=frozenset()`. A `database` grant was therefore ALL-OR-NOTHING — every row,
73
+ #: every column — and this registry was the only wall behind it.
74
+ #:
75
+ #: ⭐ WHAT CHANGED: `perm_scope.scoped_table` (contract C1) is the ONE door to any database's
76
+ #: rows. `routes_tables` applies the permanent filter before `pids` is taken and the transitive
77
+ #: hidden-field closure after `workspace_wire`, on EVERY `ut_*` read — the same code that walls
78
+ #: `customer_data` — and a door that cannot apply them REFUSES rather than serving the lot. So a
79
+ #: `database` grant is once again bounded by a second wall: it decides WHETHER an account reaches
80
+ #: the database, and C1 decides WHICH rows and columns it then sees. An `*` grant still admits
81
+ #: every account in the tenant, and each of them still only sees what their own wall allows.
82
+ #:
83
+ #: ⚠ `routes_shares.py`'s module docstring carries the OLD sentence at the other door and is in
84
+ #: no wave-36 fence — the audit's own fix was "say so at both", so one of the two is now stale.
85
+ #: Booked in `mailbox/C.md` (C-14) rather than edited across a fence
86
+ #: [[two-gates-can-assert-opposite-things]].
87
+ EVERYONE = '*'
88
+
89
+ ROLES = ('view', 'edit')
90
+
91
+ #: ⛔⛔ A FIELD's OBJECT ID IS TOPIC-QUALIFIED, AND THE SEPARATOR IS DECLARED HERE SO THERE IS ONE
92
+ #: SPELLING OF IT. A bare field key is NOT unique: `notes` exists on a dozen databases, and a
93
+ #: grant stored under it would admit a grantee to every `notes` column in the tenant at once —
94
+ #: the widening direction, silently, forever. `table_key` is the qualifier because it is the same
95
+ #: identifier `shared_overlay.bucket()` already keys the values by, so the grant and the data it
96
+ #: governs are named by the same string ([[one-question-two-normalizers]]).
97
+ #:
98
+ #: ⚠ A `ut_*` key and a registry topic key both match `[a-z0-9_]+` and neither can contain `:`,
99
+ #: so the split below is unambiguous in both directions.
100
+ FIELD_OID_SEP = ':'
101
+
102
+
103
+ def field_oid(table_key, field_key):
104
+ """`"<table_key>:<field_key>"` — the share id of ONE column on ONE database."""
105
+ table = str(table_key or '').strip()
106
+ field = str(field_key or '').strip()
107
+ if not table or not field:
108
+ raise ValueError('shares.field_oid: a field share names BOTH a database and a column. '
109
+ 'A bare field key repeats across tables and would grant all of them')
110
+ return f'{table}{FIELD_OID_SEP}{field}'
111
+
112
+
113
+ def split_field_oid(oid):
114
+ """`(table_key, field_key)` or `(None, None)` for anything that is not a field oid.
115
+
116
+ ⛔ FAIL-CLOSED ON JUNK, like every other read here: a caller that cannot learn WHICH database
117
+ an id names must not fall back to "the one I happen to be looking at", which is how a grant
118
+ on somebody else's column would be read as a grant on this one.
119
+ """
120
+ raw = str(oid or '')
121
+ table, sep, field = raw.partition(FIELD_OID_SEP)
122
+ if not sep or not table.strip() or not field.strip() or FIELD_OID_SEP in field:
123
+ return (None, None)
124
+ return (table.strip(), field.strip())
125
+
126
+
127
+ def _st(st):
128
+ return st if st is not None else store
129
+
130
+
131
+ def _check_kind(kind):
132
+ k = str(kind or '').strip().lower()
133
+ if k not in KINDS:
134
+ raise ValueError(f'{kind!r} is not a shareable kind. Use one of {", ".join(KINDS)}. '
135
+ f'This door refuses to invent a namespace from a typo.')
136
+ return k
137
+
138
+
139
+ def _clean_entries(entries):
140
+ """Normalise + REJECT junk, returning [{'user': str, 'role': 'view'|'edit'}].
141
+
142
+ Silently dropping a malformed entry is right here and wrong elsewhere: the caller is a UI
143
+ that just listed the people it is about to grant, so a rejected row must not abort the whole
144
+ save — but an entry with an unknown ROLE must not be stored as something else's default
145
+ either. Dropped, never coerced.
146
+ """
147
+ out, seen = [], set()
148
+ for e in entries or ():
149
+ if not isinstance(e, dict):
150
+ continue
151
+ user = str(e.get('user') or '').strip().lower()
152
+ role = str(e.get('role') or '').strip().lower()
153
+ if not user or role not in ROLES or user in seen:
154
+ continue
155
+ seen.add(user)
156
+ out.append({'user': user, 'role': role})
157
+ return out
158
+
159
+
160
+ def grants(kind, oid, st=None):
161
+ """`{'owner': str|None, 'entries': [{'user','role'}]}` — never raises on a junk bucket."""
162
+ kind = _check_kind(kind)
163
+ try:
164
+ bucket = (_st(st).get(SHARES_KEY) or {}).get(kind) or {}
165
+ rec = bucket.get(str(oid)) or {}
166
+ except Exception:
167
+ return {'owner': None, 'entries': []}
168
+ if not isinstance(rec, dict):
169
+ return {'owner': None, 'entries': []}
170
+ return {'owner': (str(rec.get('owner')).strip().lower() if rec.get('owner') else None),
171
+ 'entries': _clean_entries(rec.get('entries'))}
172
+
173
+
174
+ def set_grants(kind, oid, entries, owner=None, st=None):
175
+ """REPLACE the grant set for one object. Returns the stored record.
176
+
177
+ ⚠ REPLACE, NOT MERGE, and that is the contract the UI needs: revoking is expressed by an
178
+ entry's ABSENCE. A merge-only API cannot remove anybody without a second verb, and the
179
+ manage-access editor R10 asks for is exactly "here is the list now".
180
+ """
181
+ kind = _check_kind(kind)
182
+ oid = str(oid)
183
+ clean = _clean_entries(entries)
184
+ owner_l = str(owner).strip().lower() if owner else None
185
+
186
+ def _apply(data):
187
+ by_kind = dict(data.get(kind) or {})
188
+ prior = by_kind.get(oid) if isinstance(by_kind.get(oid), dict) else {}
189
+ # The owner is STICKY: set once, and a later save that omits it must not orphan the
190
+ # object. An ownerless grant record cannot answer "who may re-share this", so every
191
+ # administer check would fail closed and the object would become unmanageable.
192
+ keep_owner = owner_l or (str(prior.get('owner')).strip().lower()
193
+ if prior.get('owner') else None)
194
+ if not clean and not keep_owner:
195
+ by_kind.pop(oid, None) # fully un-shared and unowned: leave no empty husk
196
+ else:
197
+ by_kind[oid] = {'owner': keep_owner, 'entries': clean}
198
+ data[kind] = by_kind
199
+ return data
200
+
201
+ _st(st).update(SHARES_KEY, _apply, flush='async')
202
+ return grants(kind, oid, st=st)
203
+
204
+
205
+ def role_for(kind, oid, user, is_admin=False, st=None):
206
+ """`'owner'` | `'edit'` | `'view'` | `None` — the caller's effective role, fail-closed.
207
+
208
+ An ADMIN reads as `'owner'`: an admin who could not administer an object could not
209
+ administer the tenant either, which is `table_store._may_administer`'s existing rule and is
210
+ kept identical here so the two cannot disagree about the same view.
211
+ """
212
+ user = str(user or '').strip().lower()
213
+ if not user:
214
+ return None
215
+ rec = grants(kind, oid, st=st)
216
+ if is_admin or (rec['owner'] and rec['owner'] == user):
217
+ return 'owner'
218
+ best = None
219
+ for e in rec['entries']:
220
+ if e['user'] == user or e['user'] == EVERYONE:
221
+ # The STRONGER of the two wins when both a personal and an everyone grant exist:
222
+ # naming somebody explicitly is how you RAISE them above the room, so an
223
+ # everyone-view + alice-edit pair must leave alice editing.
224
+ if e['role'] == 'edit':
225
+ return 'edit'
226
+ best = best or 'view'
227
+ return best
228
+
229
+
230
+ def may_see(kind, oid, user, is_admin=False, st=None):
231
+ return role_for(kind, oid, user, is_admin=is_admin, st=st) is not None
232
+
233
+
234
+ def may_edit(kind, oid, user, is_admin=False, st=None):
235
+ return role_for(kind, oid, user, is_admin=is_admin, st=st) in ('owner', 'edit')
236
+
237
+
238
+ def may_administer(kind, oid, user, is_admin=False, st=None):
239
+ """Only the owner or an admin may change grants or delete. See the module note on why this
240
+ is deliberately narrower than `may_edit`."""
241
+ return role_for(kind, oid, user, is_admin=is_admin, st=st) == 'owner'
242
+
243
+
244
+ def shared_with(user, kind=None, st=None):
245
+ """Every object id this user has been granted (excluding what they own).
246
+
247
+ This is the "Shared with me" query (R10). It EXCLUDES owned objects deliberately: a folder
248
+ you made is not something shared *with* you, and listing it there would make the system
249
+ folder a duplicate of the rail above it.
250
+ """
251
+ user = str(user or '').strip().lower()
252
+ if not user:
253
+ return {}
254
+ try:
255
+ data = _st(st).get(SHARES_KEY) or {}
256
+ except Exception:
257
+ return {}
258
+ out = {}
259
+ for k in ([_check_kind(kind)] if kind else KINDS):
260
+ hits = []
261
+ for oid, rec in (data.get(k) or {}).items():
262
+ if not isinstance(rec, dict):
263
+ continue
264
+ owner = str(rec.get('owner') or '').strip().lower()
265
+ if owner == user:
266
+ continue
267
+ for e in _clean_entries(rec.get('entries')):
268
+ if e['user'] in (user, EVERYONE):
269
+ hits.append(str(oid))
270
+ break
271
+ out[k] = sorted(hits)
272
+ return out if kind is None else {_check_kind(kind): out[_check_kind(kind)]}
273
+
274
+
275
+ def drop_objects(pairs, st=None):
276
+ """Remove whole grant RECORDS, owner husk included — wave 21, item 6a (C3).
277
+
278
+ A deleted object's grants must die with it: `shared_with` would otherwise serve ghost ids
279
+ into every receiver's "Shared with me" forever, and the ghost would 404 on open. One
280
+ transaction for the whole sweep — a table delete drops its database grant plus a view
281
+ grant per view that lived in its bucket."""
282
+ want = {}
283
+ for kind, oid in pairs or ():
284
+ want.setdefault(_check_kind(kind), set()).add(str(oid))
285
+ if not want:
286
+ return
287
+
288
+ def _apply(data):
289
+ for kind, oids in want.items():
290
+ by_kind = data.get(kind)
291
+ if isinstance(by_kind, dict):
292
+ for oid in oids:
293
+ by_kind.pop(oid, None)
294
+ return data
295
+
296
+ _st(st).update(SHARES_KEY, _apply, flush='async')
platform/core/store.py CHANGED
The diff for this file is too large to render. See raw diff
 
platform/core/table_store.py CHANGED
@@ -1,797 +1,797 @@
1
- """The generic per-user TABLE WORKSPACE store — the persistence half of the table-page factory.
2
-
3
- One durable store key holds one table OBJECT's per-user Airtable-style state:
4
-
5
- {username: {'views': {view_id: SavedView},
6
- 'fields': {field_key: Field}, # notes + custom_ + measure_ strata
7
- 'overlays': {str(pid): {field_key: value}}}}
8
-
9
- `make(table_key)` returns the six operations a table page's host loop needs, closed over that
10
- key. The Customer table's ops (`modules/customer_data.py`, key 'customer_table_workspace') are
11
- these exact functions — the logic MOVED here 2026-07-27 so that duplicating the Customer table
12
- pattern to a new object is a registry row + a config, not a copy of the store plumbing
13
- (owner directive: the table-page factory).
14
-
15
- A LIST (membership/formula semantics) is deliberately a different store from a VIEW
16
- (presentation/query state) — see modules/customer_data.py's customer_lists key.
17
- """
18
- import core.store as store
19
-
20
- #: Wave-9 I17 — the SHARED bucket. Views whose permissions make them visible to anyone but
21
- #: their creator live here instead of in a personal workspace, under a key that cannot collide
22
- #: with a username (usernames come from core/users.py and are never dunder-wrapped; `is_shared`
23
- #: guards it anyway). ONE HOME PER VIEW, never both: a view moved to personal is REMOVED from
24
- #: here, and a view shared is removed from its creator's workspace. Two homes would mean two
25
- #: divergent copies the moment either was edited.
26
- SHARED_KEY = '__shared__'
27
-
28
- #: ⭐⭐ W36-T25 — THE COLUMN SUMMARY IS THE DATABASE'S, NOT ONE ACCOUNT'S.
29
- #:
30
- #: Owner item 1, second half, verbatim: *"When i filter Avi on my computer sum works for the Sales
31
- #: last 365 days column but not when my colleague filter it — The sum of the field is not showing
32
- #: at the bottom even when i zoom out. It only works on my screen??"* Measured while scouting it:
33
- #: `save_field` writes the WHOLE field payload into `data[username]`, so a column summary — which
34
- #: is a statement about the COLUMN, identical for every reader by construction — was stored once
35
- #: per account. Set it, and the totals row exists for you and for nobody else. `computeAggs` then
36
- #: paints no totals row at all for the colleague (`showTotals` is false when no field carries an
37
- #: `agg`), which is exactly "not showing at the bottom".
38
- #:
39
- #: ⛔ ONLY `agg` MOVES, AND THE LINE IS NOT ARBITRARY. Width, column order, the note, the display
40
- #: format and every `custom_`/`measure_` definition stay per user, because each of those is a
41
- #: statement about how ONE PERSON reads the column. "Sum this column" is a statement about what
42
- #: the column MEANS, and two accounts disagreeing about it is the defect, not a preference.
43
- #:
44
- #: ⚠ IT LIVES IN THE `__shared__` MEMBER OF THIS SAME BUCKET rather than in
45
- #: `core/shared_overlay.py`'s `<key>__shared` document, and the reason is transactional: a summary
46
- #: is written by the same `save_field` call that writes the note beside it, and the two must land
47
- #: or fail together. `__shared__` is already this store's tenant-wide member (shared VIEWS live
48
- #: there) and is guarded against colliding with a username by `is_shared` and by `core/users.py`'s
49
- #: never-dunder rule, so there is no second bucket, no second flush and no second failure mode.
50
- SHARED_FIELD_KEYS = ('agg',)
51
-
52
-
53
- def _may_see(view, viewer, is_admin=False):
54
- """Visibility for ONE shared view, fail-closed.
55
-
56
- 'collaborative' = everyone who can already open the module (the caller has gated that).
57
- 'users' = the named users, plus the creator, plus admins — an admin who could not
58
- see a view could not administer it either.
59
- Anything unrecognised returns False rather than defaulting open: an unreadable permission
60
- must never widen access ([[aios-permissioning]] — no fail-open defaults).
61
- """
62
- if not isinstance(view, dict):
63
- return False
64
- if view.get('createdBy') == viewer or is_admin:
65
- return True
66
- perms = view.get('permissions') or {}
67
- edit = perms.get('edit')
68
- if edit == 'collaborative':
69
- return True
70
- if edit == 'users':
71
- return viewer in set(perms.get('users') or ())
72
- return False # 'personal', absent, or junk
73
-
74
-
75
- def _may_edit(view, viewer, is_admin=False):
76
- """Who may WRITE a shared view. Same set as visibility today — the owner's item asks 'who
77
- can edit' and lists who 'can have access', i.e. seeing and editing are one grant. Kept as a
78
- separate function so they can diverge (a future read-only share) without hunting callers."""
79
- return _may_see(view, viewer, is_admin)
80
-
81
-
82
- def _may_administer(view, viewer, is_admin=False):
83
- """Who may change a view's PERMISSIONS, or delete it: the creator or an admin ONLY.
84
-
85
- Deliberately narrower than _may_edit. If a collaborator could rewrite `permissions` they
86
- could grant themselves sole ownership of somebody else's view, or quietly widen a
87
- users-scoped view to everyone — the classic privilege-escalation-by-edit hole.
88
- """
89
- if not isinstance(view, dict):
90
- return False
91
- return bool(is_admin) or view.get('createdBy') == viewer
92
-
93
-
94
- def is_shared(view):
95
- """A view belongs in the shared bucket when its permissions reach beyond its creator."""
96
- return ((view or {}).get('permissions') or {}).get('edit') in ('collaborative', 'users')
97
-
98
-
99
- def _shared_fields(data):
100
- """The tenant-wide field stratum of one workspace document — `{field_key: {'agg': …}}`.
101
-
102
- ⚠ TOTAL AND FAIL-SOFT: a document written before W36-T25 has no `__shared__` member at all,
103
- and one written by a future version may have something else in it. Either way this answers an
104
- empty mapping rather than raising, because the caller is a READ that must still serve the
105
- workspace — a column with no summary is the state every column was in yesterday.
106
- """
107
- shared = (data or {}).get(SHARED_KEY)
108
- fields = (shared or {}).get('fields') if isinstance(shared, dict) else None
109
- return fields if isinstance(fields, dict) else {}
110
-
111
-
112
- def split_shared(payload):
113
- """`(per_user, shared)` — one field payload divided at the strata boundary.
114
-
115
- ⭐ W36-T25. `shared` carries only `SHARED_FIELD_KEYS` that are actually SET; `per_user` is the
116
- payload without them. Split HERE rather than at each write door so the three doors that store
117
- a field (`save_field`, `duplicate_field`, and the delete that must clear it) cannot come apart
118
- about where a summary lives — which is the whole reason this module exists rather than being
119
- copied per topic.
120
- """
121
- payload = dict(payload or {})
122
- shared = {}
123
- for key in SHARED_FIELD_KEYS:
124
- value = str(payload.pop(key, '') or '').strip()
125
- if value:
126
- shared[key] = value
127
- return payload, shared
128
-
129
-
130
- def source_override_is_empty(payload):
131
- """Does this stored definition of a SOURCE (non-custom) column carry any user state?
132
-
133
- A cleared note on an immutable source field returns to the canonical schema instead of
134
- leaving a meaningless override row. Custom fields remain even with an empty note — and so
135
- does a PRESET field carrying a measure-window override (wave-2 item 8), a DISPLAY-format
136
- override (wave-5 item 10), and, since W29-T83, a COLUMN SUMMARY.
137
-
138
- ⛔ EVERY CLAUSE IS A SETTING A USER MADE, and each one omitted is a setting that silently
139
- stops surviving a session. `agg` was missing: choosing Average on Customer's `Overdue days`
140
- built an override whose only content was that summary, so this rule threw the whole row away
141
- on write while the menu went on reading "Summary: Average" from the client's own optimistic
142
- copy until the next login — a discarded WRITE wearing the face of a failed read
143
- ([[lost-write-looks-like-failed-read]]). Measured on `bac40c2`; a `ut_*` table, which stores
144
- its definitions through another door entirely, kept it.
145
-
146
- ⚠ ONE RULE, TWO CALLERS — here and `grid_events`' store-less fallback. Two copies of a
147
- discard rule is how one of them keeps a write the other bins ([[one-evaluator-per-question]]).
148
- ⚠ A CLEARED summary still drops the row, which is the intent: with nothing else set, the
149
- column goes back to whatever the contract declares for it.
150
- """
151
- payload = payload or {}
152
- if payload.get('custom'):
153
- return False
154
- return (not str(payload.get('note') or '').strip()
155
- and not isinstance(payload.get('measure'), dict)
156
- and not isinstance(payload.get('format'), dict)
157
- and not str(payload.get('agg') or '').strip())
158
-
159
-
160
- def _unique_name(wanted, existing, *, fallback='Untitled', max_len=120):
161
- """Allocate one human-facing name inside a store.update transaction.
162
-
163
- Keys/ids remain structural identity. Names compare case-insensitively after collapsing
164
- whitespace, because those variants are indistinguishable in the UI. This helper belongs
165
- in the store layer: allocating from a pre-write snapshot lets two concurrent requests both
166
- choose the same free name before either write lands.
167
- """
168
- limit = max(1, int(max_len))
169
-
170
- def _clean(value):
171
- return ' '.join(str(value or '').split())
172
-
173
- base = (_clean(wanted) or _clean(fallback) or 'Untitled')[:limit].rstrip()
174
- taken = {_clean(value).casefold() for value in existing if _clean(value)}
175
- if base.casefold() not in taken:
176
- return base
177
- index = 2
178
- while True:
179
- suffix = f' {index}'
180
- stem = base[:max(0, limit - len(suffix))].rstrip()
181
- candidate = f'{stem}{suffix}' if stem else str(index)[-limit:]
182
- if candidate.casefold() not in taken:
183
- return candidate
184
- index += 1
185
-
186
-
187
- class TableStore:
188
- """The six store operations for one table object's workspace, closed over its store key.
189
-
190
- `st` (wave 18, C3-UT) is the STORE HANDLE — anything exposing `get(name)` /
191
- `update(name, fn, flush=)`. Default = `core.store` (tenant #0, every existing caller,
192
- zero behaviour change). The API passes the session's `TenantRuntime`, whose accessors
193
- apply the tenant prefix / repo binding — which is what makes a user table created by a
194
- Nurilab admin land in Nurilab's store instead of Royal's.
195
- """
196
-
197
- def __init__(self, table_key, st=None):
198
- self.table_key = table_key
199
- self._st = st if st is not None else store
200
-
201
- @property
202
- def st(self):
203
- """The bound store handle — for SIBLING registries (core/shares) that must read the
204
- same tenant's buckets this workspace lives in (wave 21, C1)."""
205
- return self._st
206
-
207
- def find_view(self, view_id):
208
- """`(owner_username, view)` for a view living in ANY personal stratum, else None.
209
-
210
- ⭐ Wave 21 (item 9, C1): the R10 grant registry names bare ids, so projecting a granted
211
- view means locating the OWNER's record inside this topic's bucket. Personal strata
212
- only — the `__shared__` bucket has its own read path (`shared_views`), and serving one
213
- view from two finders is how two copies drift."""
214
- vid = str(view_id or '').strip()
215
- if not vid:
216
- return None
217
- try:
218
- data = self._st.get(self.table_key) or {}
219
- except Exception:
220
- return None
221
- for username, ws in data.items():
222
- if username == SHARED_KEY or not isinstance(ws, dict):
223
- continue
224
- v = (ws.get('views') or {}).get(vid)
225
- if isinstance(v, dict):
226
- return str(username), dict(v)
227
- return None
228
-
229
- def find_folder(self, folder_id):
230
- """`(owner_username, folder_row, {view_id: view})` for a VIEWS folder living in any
231
- personal stratum, else None. `find_view`'s sibling, and here for the same reason.
232
-
233
- ⭐ D-37 (wave 20's R10 remainder, closed 2026-08-05): the grant registry accepts kind
234
- `folder` and has since wave 20, but only the VIEW kind was ever projected — so "share
235
- this folder with Karen" recorded a row, listed under Shared with me, and put nothing on
236
- Karen's screen. Projecting a folder means two lookups the view path does not need: WHO
237
- owns it, and WHICH views are filed in it. Folder membership lives in the owner's
238
- `itemFolders` map (item id -> folder id), never on the view record, so the views are
239
- found by asking that map rather than by reading a list off the folder.
240
-
241
- Views only (`folders['views']`): the cohort surface has its own store and its own
242
- sharing question, and answering both here would make one function mean two things.
243
- """
244
- fid = str(folder_id or '').strip()
245
- if not fid:
246
- return None
247
- try:
248
- data = self._st.get(self.table_key) or {}
249
- except Exception:
250
- return None
251
- for username, ws in data.items():
252
- if username == SHARED_KEY or not isinstance(ws, dict):
253
- continue
254
- rows = (ws.get('folders') or {}).get('views') or []
255
- hit = next((f for f in rows
256
- if isinstance(f, dict) and str(f.get('id') or '') == fid), None)
257
- if not hit:
258
- continue
259
- # ⚠ `itemFolders` IS KEYED BY SURFACE FIRST (`{'views': {itemId: folderId}, …}`) —
260
- # reading item ids off the top level finds the surface names instead and matches
261
- # nothing, so the projection silently returns an EMPTY folder and the feature looks
262
- # exactly as broken as it was before the fix. Caught by this change's own gate,
263
- # which is the entire argument for writing one.
264
- placed = (ws.get('itemFolders') or {}).get('views') or {}
265
- views = ws.get('views') or {}
266
- inside = {str(vid): dict(v) for vid, v in views.items()
267
- if isinstance(v, dict) and str(placed.get(str(vid)) or '') == fid}
268
- return str(username), dict(hit), inside
269
- return None
270
-
271
- # ---------------------------------------------------------------- read
272
- def workspace(self, username, consume_corrections=True):
273
- """One user's durable workspace: always the full three-strata shape."""
274
- try:
275
- data = self._st.get(self.table_key) or {}
276
- ws = data.get(username, {}) or {}
277
- # A collision acknowledgement is protocol state, not part of a field definition.
278
- # Consume it with the first fresh workspace payload after the correcting write,
279
- # then splice a bounded copy into that payload only. Keeping it out of `fields`
280
- # prevents an old request id surviving forever and overriding a later rename.
281
- corrections = {}
282
- if consume_corrections and ws.get('fieldCorrections'):
283
- def _take(current):
284
- current_ws = current.get(username) or {}
285
- pending = current_ws.get('fieldCorrections') or {}
286
- corrections.update({
287
- str(key)[:80]: dict(value)
288
- for key, value in pending.items()
289
- if isinstance(value, dict)
290
- })
291
- current_ws.pop('fieldCorrections', None)
292
- return current
293
-
294
- data = self._st.update(self.table_key, _take, flush='async')
295
- ws = (data or {}).get(username, {}) or {}
296
- except Exception:
297
- data = {}
298
- ws = {}
299
- corrections = {}
300
- fields = {
301
- key: dict(value) if isinstance(value, dict) else value
302
- for key, value in (ws.get('fields') or {}).items()
303
- }
304
- # ⭐⭐ W36-T25 — THE TENANT-WIDE COLUMN SUMMARY, MERGED OVER THIS USER'S STRATUM.
305
- # ⛔ IT MUST BE ABLE TO CREATE AN ENTRY, not only decorate one, and that is the whole
306
- # reason this is a merge rather than a lookup: the colleague who never touched the column
307
- # has NO per-user record for it, so a decorate-only pass would have left them with exactly
308
- # the blank totals row the owner reported. `aios_grid.workspace_wire` reads `meta['agg']`
309
- # off whatever is here, base column or custom one alike.
310
- for key, shared in _shared_fields(data).items():
311
- agg = str((shared or {}).get('agg') or '').strip()
312
- if not agg:
313
- continue
314
- entry = fields.get(key)
315
- fields[key] = {**entry, 'agg': agg} if isinstance(entry, dict) else {'agg': agg}
316
- for key, ack in corrections.items():
317
- field = fields.get(key)
318
- accepted_label = str(ack.get('label') or '')[:120]
319
- requested_label = str(ack.get('labelCorrectedFrom') or '')[:120]
320
- correction_id = str(ack.get('labelCorrectionId') or '')[:180]
321
- # A newer field write clears/replaces the pending ack in the SAME transaction.
322
- # The label check is an extra belt against ever attaching a stale ack to a newer
323
- # definition if a future store implementation weakens that ordering.
324
- if (isinstance(field, dict) and accepted_label
325
- and str(field.get('label') or '') == accepted_label
326
- and requested_label and correction_id):
327
- field['labelCorrectedFrom'] = requested_label
328
- field['labelCorrectionId'] = correction_id
329
- out = {
330
- 'views': dict(ws.get('views') or {}),
331
- 'fields': fields,
332
- 'overlays': dict(ws.get('overlays') or {}),
333
- # wave-8 I11 (C4): folders over the saved views / cohorts sidebars. A FOURTH
334
- # stratum rather than a key on each item — see aios_grid.clean_folders for why
335
- # (a cohort lives in another store, and filing is an organising act, not part of
336
- # what a view is). Absent for every workspace saved before this wave, which is
337
- # exactly "no folders yet".
338
- 'folders': dict(ws.get('folders') or {}),
339
- 'itemFolders': dict(ws.get('itemFolders') or {}),
340
- }
341
- # 2026-07-31 (owner item 3): WHERE THE USER LEFT OFF survives a new browser. The
342
- # client's localStorage copy wins when present; this is the server's answer for a
343
- # fresh profile, which used to fall all the way to the system default view.
344
- if ws.get('activeViewId'):
345
- out['activeViewId'] = str(ws['activeViewId'])
346
- # Wave 2026-08-02 (C-LAYOUT): the per-user record-detail field order. A fifth
347
- # stratum, absent until the user first reorders — exactly "default order".
348
- if isinstance(ws.get('recordLayout'), dict):
349
- out['recordLayout'] = dict(ws['recordLayout'])
350
- return out
351
-
352
- # ---------------------------------------------------------------- write
353
- def _update(self, username, change, shared=None):
354
- """Apply `change(ws)` to this user's stratum, and `shared(shared_fields)` to the
355
- tenant-wide one, in ONE transaction.
356
-
357
- ⭐ W36-T25 — `shared` IS A SECOND CALLBACK RATHER THAN A SECOND `update`, and that is the
358
- whole reason it exists here instead of at the caller. `save_field` writes a note (per
359
- user) and a column summary (tenant-wide) from ONE payload; two transactions would let the
360
- summary land while the note did not, and the store's own commit is asynchronous, so the
361
- window is real rather than theoretical. One `update`, one flush, one failure mode.
362
- """
363
- def _up(data):
364
- ws = data.setdefault(username, {})
365
- ws.setdefault('views', {})
366
- ws.setdefault('fields', {})
367
- ws.setdefault('overlays', {})
368
- ws.setdefault('folders', {})
369
- ws.setdefault('itemFolders', {})
370
- change(ws)
371
- if shared is not None:
372
- shared(data.setdefault(SHARED_KEY, {}).setdefault('fields', {}))
373
- return data
374
- # flush='async' (wave-7 W3): this is THE hot path — every autosaved filter tweak,
375
- # column note and typed overlay cell lands here inside the component round-trip, and
376
- # the historical synchronous hub commit cost seconds per edit. The mutation applies to
377
- # the in-process cache (read-your-writes for every subsequent render); the hub write
378
- # coalesces in the background. Registry/auth writes elsewhere stay flush='sync'.
379
- return self._st.update(self.table_key, _up, flush='async')
380
-
381
- def rename_choice_values(self, username, change):
382
- """Apply an arbitrary workspace rewrite (wave 20, item 15 / C-RENAME).
383
-
384
- ⚠ NAMED FOR ITS ONE CALLER RATHER THAN EXPOSED AS A GENERIC `mutate`, deliberately. A
385
- public "do anything to the workspace" method is an invitation to put write logic in
386
- callers instead of here, and every OTHER method on this class exists precisely because
387
- that logic belongs in one place. Renaming a choice is the one operation that must touch
388
- three strata AT ONCE — the field's `choices`, the cells in `overlays`, and the views that
389
- filter or colour by the old value — inside a SINGLE transaction, because a rename that
390
- updated the cells and not the filters would leave a saved view matching nothing.
391
-
392
- `change(ws)` receives the whole workspace with every stratum pre-created (see `_update`).
393
- """
394
- return self._update(username, change)
395
-
396
- def save_active_view(self, username, view_id):
397
- """Remember which view this user last opened (owner item 3, 2026-07-31).
398
-
399
- Presentation state, not authorisation: the READ side re-validates the id against what
400
- the caller may actually see, so a stale or foreign id degrades to the default view
401
- rather than granting anything. Stored per user like every other stratum.
402
- """
403
- vid = str(view_id or '').strip()[:120]
404
- if not vid or username == SHARED_KEY:
405
- return
406
-
407
- def _set(ws):
408
- ws['activeViewId'] = vid
409
- self._update(username, _set)
410
-
411
- def save_record_layout(self, username, order):
412
- """The per-user RECORD-DETAIL field order (wave 2026-08-02, C-LAYOUT).
413
-
414
- Presentation state for ONE surface — the record modal. Deliberately not view config:
415
- the owner's ask is per-user, not per-view, and it must never reorder grid columns.
416
- The event handler validated keys against the live field set; the wire re-validates at
417
- serve time (aios_grid.workspace_wire), so a deleted field cannot outlive itself here.
418
- An empty order clears the stratum back to "default order".
419
- """
420
- if username == SHARED_KEY:
421
- return
422
- clean, seen = [], set()
423
- for key in (order or [])[:200]:
424
- key = str(key or '').strip()[:80]
425
- if key and key not in seen:
426
- seen.add(key)
427
- clean.append(key)
428
-
429
- def _set(ws):
430
- if clean:
431
- ws['recordLayout'] = {'order': clean}
432
- else:
433
- ws.pop('recordLayout', None)
434
-
435
- self._update(username, _set)
436
-
437
- def save_folders(self, username, folders, item_folders):
438
- """Replace the folder stratum wholesale (wave-8 I11).
439
-
440
- Wholesale rather than per-folder because the caller has ALREADY validated the complete
441
- picture through aios_grid.clean_folders / clean_item_folders, and those two are
442
- interdependent: a placement is only legal while its folder exists, so committing them
443
- separately would leave a window where a reader sees an item filed into a folder that is
444
- not there yet. One write, one consistent state.
445
- """
446
- def _set(ws):
447
- ws['folders'] = dict(folders or {})
448
- ws['itemFolders'] = dict(item_folders or {})
449
- self._update(username, _set)
450
-
451
- def save_view_order(self, username, order):
452
- """⭐ WAVE-27 item 5 (contract C7) — this user's own ORDER for the views rail.
453
-
454
- Wholesale, like `save_folders` above and for the same reason: the client sends the full
455
- list it is looking at, not a delta, because a partial order cannot say where an UNNAMED
456
- view went.
457
-
458
- ⛔ PER USER, and it belongs in this stratum rather than on the view records themselves.
459
- `aios_grid`'s own folder note argues it out for placements and every word applies: an
460
- arrangement is a per-user ORGANISING act, not part of what a view IS — so keeping it out
461
- of the view config means duplicating, sharing or exporting a view does not drag one
462
- person's rail position along with it. It also means a SHARED view can sit in a different
463
- place for each person who can see it, which is the only coherent answer once two people
464
- share one view.
465
-
466
- An empty list CLEARS the arrangement (back to server order) rather than storing `[]`.
467
- """
468
- def _set(ws):
469
- clean = []
470
- seen = set()
471
- for vid in (order or []):
472
- vid = str(vid).strip()[:120]
473
- if vid and vid not in seen:
474
- seen.add(vid)
475
- clean.append(vid)
476
- if clean:
477
- ws['viewOrder'] = clean
478
- else:
479
- ws.pop('viewOrder', None)
480
- self._update(username, _set)
481
-
482
- def shared_views(self, viewer, is_admin=False):
483
- """Every SHARED view this viewer may see, by id (wave-9 I17).
484
-
485
- Read-only and independent of the viewer's own workspace: the caller merges. Returns
486
- only what `_may_see` allows, so a caller cannot accidentally render somebody else's
487
- personal view by forgetting to filter.
488
- """
489
- try:
490
- bucket = ((self._st.get(self.table_key) or {}).get(SHARED_KEY) or {}).get('views') or {}
491
- except Exception:
492
- return {}
493
- return {vid: dict(v) for vid, v in bucket.items()
494
- if _may_see(v, viewer, is_admin)}
495
-
496
- def shared_view(self, view_id):
497
- """One shared view RAW — no visibility filter. For authorisation decisions only: a
498
- caller must know a view exists and who owns it before it can decide whether the actor
499
- may touch it. Never hand the result to a renderer without checking `_may_see`."""
500
- try:
501
- return ((self._st.get(self.table_key) or {}).get(SHARED_KEY) or {}
502
- ).get('views', {}).get(str(view_id))
503
- except Exception:
504
- return None
505
-
506
- def save_view(self, username, view, shared=None, reserved_names=(), is_admin=False):
507
- """Upsert a SavedView into its ONE home — personal workspace or the shared bucket.
508
-
509
- `shared` defaults to reading the view's own permissions (`is_shared`). Whichever home
510
- it lands in, the view is REMOVED from the other, so a view can never exist as two
511
- copies that diverge on the next edit.
512
-
513
- ⚠ AUTHORISATION IS THE CALLER'S JOB and must happen BEFORE this is called — this layer
514
- moves data and does not know who is asking. `_cl_handle_one` is the wall.
515
- """
516
- view_id = str((view or {}).get('id') or '').strip()
517
- if not view_id:
518
- raise ValueError('view id is required')
519
- if username == SHARED_KEY:
520
- raise ValueError('reserved username')
521
- to_shared = is_shared(view) if shared is None else bool(shared)
522
- requested = dict(view)
523
- accepted = {}
524
-
525
- def _up(data):
526
- # View names are tenant-global: every personal workspace plus the shared bucket.
527
- # This deliberately includes views the actor cannot see. The only disclosed fact
528
- # is that a display name is already taken, while the categorical "no duplicate
529
- # view names" contract remains true when a personal view is later shared.
530
- names = list(reserved_names or ())
531
- for workspace in data.values():
532
- if not isinstance(workspace, dict):
533
- continue
534
- names.extend(
535
- value.get('name')
536
- for candidate_id, value in (workspace.get('views') or {}).items()
537
- if candidate_id != view_id and isinstance(value, dict)
538
- )
539
- payload = dict(requested)
540
- payload['name'] = _unique_name(payload.get('name'), names)
541
- accepted.clear()
542
- accepted.update(payload)
543
- if to_shared:
544
- bucket = data.setdefault(SHARED_KEY, {})
545
- bucket.setdefault('views', {})[view_id] = payload
546
- # it may have lived in the creator's workspace before being shared
547
- owner = data.get(payload.get('createdBy') or username) or {}
548
- (owner.get('views') or {}).pop(view_id, None)
549
- else:
550
- ws = data.setdefault(username, {})
551
- ws.setdefault('views', {})[view_id] = payload
552
- (data.get(SHARED_KEY, {}).get('views') or {}).pop(view_id, None)
553
- return data
554
-
555
- self._st.update(self.table_key, _up, flush='async')
556
- return dict(accepted)
557
-
558
- def delete_view(self, username, view_id):
559
- """Delete a custom/list view override. The system all-rows view is guarded by caller.
560
-
561
- Removes from BOTH homes: the caller has already authorised the delete, and leaving a
562
- stale copy in the other bucket would resurrect the view on the next read.
563
- """
564
- vid = str(view_id)
565
-
566
- def _up(data):
567
- (data.get(username, {}).get('views') or {}).pop(vid, None)
568
- (data.get(SHARED_KEY, {}).get('views') or {}).pop(vid, None)
569
- return data
570
-
571
- self._st.update(self.table_key, _up, flush='async')
572
-
573
- def save_field(self, username, field, reserved_names=(), correction_id=None):
574
- """Persist a column note or a user-created (custom_/measure_) field definition.
575
-
576
- ⭐⭐ W36-T25 — THE COLUMN SUMMARY GOES TO THE TENANT-WIDE STRATUM, EVERYTHING ELSE STAYS
577
- PER USER, in ONE transaction. See `SHARED_FIELD_KEYS` above for the owner's report and for
578
- why the line falls where it does. The returned `accepted` still carries the summary: the
579
- caller is echoing back the field it just stored, and dropping a key from that echo would
580
- tell the client its write was refused ([[read-path-cannot-witness-write-path]]).
581
- """
582
- key = str((field or {}).get('key') or '').strip()
583
- if not key:
584
- raise ValueError('field key is required')
585
- requested = dict(field)
586
- accepted = {}
587
-
588
- def _save(ws):
589
- names = list(reserved_names or ())
590
- names.extend(
591
- value.get('label')
592
- for candidate_key, value in (ws.get('fields') or {}).items()
593
- if candidate_key != key and isinstance(value, dict)
594
- )
595
- payload = dict(requested)
596
- payload.pop('labelCorrectedFrom', None)
597
- payload.pop('labelCorrectionId', None)
598
- requested_label = ' '.join(
599
- str(payload.get('label') or 'Untitled').split())[:120].rstrip()
600
- payload['label'] = _unique_name(requested_label, names)
601
- corrections = ws.setdefault('fieldCorrections', {})
602
- corrections.pop(key, None)
603
- if payload['label'] != requested_label and correction_id:
604
- corrections[key] = {
605
- 'label': payload['label'],
606
- 'labelCorrectedFrom': requested_label,
607
- 'labelCorrectionId': str(correction_id)[:180],
608
- }
609
- if not corrections:
610
- ws.pop('fieldCorrections', None)
611
- accepted.clear()
612
- accepted.update(payload)
613
- # ⭐⭐ W36-T25 — SPLIT AFTER the label allocation and the correction bookkeeping, so
614
- # both still see the whole payload, and BEFORE the per-user write.
615
- mine, shared_now = split_shared(payload)
616
- shared_write.clear()
617
- shared_write.update(shared_now)
618
- # ⚠ EMPTINESS IS JUDGED ON THE PER-USER HALF. A source column whose ONLY state was a
619
- # summary now has no per-user state at all, and leaving an `{}` override behind is the
620
- # meaningless row `source_override_is_empty` exists to prevent.
621
- if source_override_is_empty(mine):
622
- ws['fields'].pop(key, None)
623
- else:
624
- ws['fields'][key] = mine
625
-
626
- shared_write = {}
627
-
628
- def _share(shared_fields):
629
- # ⛔ A CLEARED SUMMARY MUST REMOVE THE ROW, not leave an empty one: `workspace` treats
630
- # any entry it finds as a live tenant-wide summary, so an `{'agg': ''}` husk would be
631
- # skipped today and become a resurrection hazard the moment the read grows a second
632
- # shared key. Absence is the only honest spelling of "nobody set one".
633
- if shared_write:
634
- shared_fields[key] = {**(shared_fields.get(key) or {}), **shared_write}
635
- else:
636
- shared_fields.pop(key, None)
637
-
638
- self._update(username, _save, shared=_share)
639
- return dict(accepted)
640
-
641
- def duplicate_field(self, username, source_key, new_key, field,
642
- reserved_names=(), correction_id=None):
643
- """Clone a user-created field in ONE store transaction (wave-5 item 1): the new
644
- definition plus — for `custom_` overlay sources only — every stored cell value under
645
- the source key. One transaction, because a def without its values (or values without a
646
- def) is exactly the orphan state delete_field exists to prevent, in reverse.
647
- The caller validated both keys (same created stratum) and stamped the clone's
648
- createdBy; this layer only moves data."""
649
- source_key = str(source_key or '').strip()
650
- new_key = str(new_key or '').strip()
651
- if not source_key or not new_key or source_key == new_key:
652
- raise ValueError('duplicate_field needs two distinct keys')
653
- requested = dict(field)
654
- accepted = {}
655
-
656
- def _dup(ws):
657
- names = list(reserved_names or ())
658
- names.extend(
659
- value.get('label')
660
- for candidate_key, value in (ws.get('fields') or {}).items()
661
- if candidate_key != new_key and isinstance(value, dict)
662
- )
663
- payload = dict(requested)
664
- payload.pop('labelCorrectedFrom', None)
665
- payload.pop('labelCorrectionId', None)
666
- requested_label = ' '.join(
667
- str(payload.get('label') or 'Untitled').split())[:120].rstrip()
668
- payload['label'] = _unique_name(requested_label, names)
669
- corrections = ws.setdefault('fieldCorrections', {})
670
- corrections.pop(new_key, None)
671
- if payload['label'] != requested_label and correction_id:
672
- corrections[new_key] = {
673
- 'label': payload['label'],
674
- 'labelCorrectedFrom': requested_label,
675
- 'labelCorrectionId': str(correction_id)[:180],
676
- }
677
- if not corrections:
678
- ws.pop('fieldCorrections', None)
679
- accepted.clear()
680
- accepted.update(payload)
681
- # ⭐ W36-T25: a CLONE carries the original's summary, and a summary is the database's
682
- # (see `SHARED_FIELD_KEYS`). Storing it per user here would give the clone a different
683
- # residency from every other column — one door out of three disagreeing about where a
684
- # thing lives is how `save_field` and this function drift.
685
- mine, shared_now = split_shared(payload)
686
- shared_write.clear()
687
- shared_write.update(shared_now)
688
- ws['fields'][new_key] = mine
689
- if source_key.startswith('custom_'):
690
- for row in ws['overlays'].values():
691
- if isinstance(row, dict) and source_key in row:
692
- row[new_key] = row[source_key]
693
-
694
- shared_write = {}
695
-
696
- def _share(shared_fields):
697
- if shared_write:
698
- shared_fields[new_key] = {**(shared_fields.get(new_key) or {}), **shared_write}
699
- else:
700
- shared_fields.pop(new_key, None)
701
-
702
- self._update(username, _dup, shared=_share)
703
- return dict(accepted)
704
-
705
- def delete_field(self, username, key):
706
- """Delete a USER-CREATED field definition outright (owner gap closed 2026-07-27).
707
-
708
- Only the created strata ever reach here (`custom_` overlay fields, `measure_` formula
709
- columns — the caller enforces the prefix). The stored overlay VALUES for the key are
710
- scrubbed with it: a deleted column's cells must not linger as orphan data that would
711
- silently resurface if the key were ever reused. Views referencing the key self-heal on
712
- their next autosave (an unknown colId is dropped) — the rule every stale key rides.
713
-
714
- ⭐⭐ W36-T25 — AND THE TENANT-WIDE SUMMARY GOES WITH IT, for exactly the reason the
715
- paragraph above gives about cells: an orphan `agg` under a deleted key is state nobody can
716
- see and nobody can clear, and it would attach itself to the next column that happens to
717
- take the key back. ⚠ This is the ONE stratum a per-user delete may reach across accounts,
718
- and it is safe because the summary was never this user's to begin with — deleting the
719
- COLUMN is a tenant-wide act already.
720
- """
721
- key = str(key or '').strip()
722
- if not key:
723
- return
724
-
725
- def _drop(ws):
726
- ws['fields'].pop(key, None)
727
- for row in ws['overlays'].values():
728
- if isinstance(row, dict):
729
- row.pop(key, None)
730
-
731
- self._update(username, _drop, shared=lambda shared_fields: shared_fields.pop(key, None))
732
-
733
- def _tenant_wide_keys(self):
734
- """The columns of THIS table whose values live in the tenant-wide stratum.
735
-
736
- ⭐ THE CHEAP HALF. A table that shares nothing reads one small per-key file (cache-first
737
- per process) and answers the empty set, so `patch_overlay` behaves exactly as it did
738
- before W38-T20 for every database that has no shared column. It is deliberately not
739
- memoised on the instance: `modules/customer_data.py` and `modules/product_data.py` both
740
- hold a MODULE-LEVEL `TABLE_OPS`, so a per-instance cache would serve one request's answer
741
- to the next, and this one decides WHERE a value is written.
742
- ⚠ LENIENT LIKE EVERY OTHER STRATUM READ, and the failure direction is the safe one: an
743
- unreachable shared bucket routes the write to the PER-USER stratum, which is the
744
- pre-ticket behaviour, rather than dropping it.
745
- """
746
- try:
747
- import core.shared_overlay as shared_overlay
748
- return set(shared_overlay.fields(self.table_key, st=self._st) or ())
749
- except Exception: # noqa: BLE001
750
- return set()
751
-
752
- def patch_overlay(self, username, pid, updates):
753
- """Patch only the external editable stratum; never writes to the source system.
754
-
755
- ⭐⭐ W38-T20 / D-423 — A CELL IN A TENANT-WIDE COLUMN GOES TO THE TENANT-WIDE STRATUM,
756
- AND WITHOUT THIS SPLIT THE EDIT SILENTLY DISAPPEARS. The read path layers the shared
757
- stratum OVER the per-user one (it has to: that is what makes every reader see the same
758
- number). This method wrote PER USER. So the sequence was: type a new value, see it accept,
759
- come back, and read the shared value again — the owner's *"I went back and it all got
760
- reseted"*, with a successful 200 at every step and nothing in any log.
761
-
762
- ⛔ IT IS DECIDED BY WHERE THE COLUMN LIVES, NOT BY WHO IS WRITING OR THROUGH WHICH ROUTE.
763
- `modules/product_data._ProductTableStore` has done exactly this since W30-T36 against its
764
- CANONICAL list; the only reason it needed a subclass is that its shared columns are
765
- declared in a contract file. Columns created at runtime cannot be, so the general form
766
- asks the stratum itself. Both doors (`PATCH /customers/{pid}` and `POST /grid/events`)
767
- arrive here through `grid_events._tops(ctx)`, which is why the split belongs at the STORE
768
- and not at either route: intercepting at one leaves the other writing into the shadow.
769
-
770
- ⛔ NO PERMISSION IS ANSWERED HERE. `shared_overlay`'s header is explicit that it is not a
771
- wall, and neither is this: whether this session may write this key is settled upstream by
772
- `EventCtx.hidden_keys`, which `routes_customers._hidden_for` now computes over the MERGED
773
- contract precisely so a grant-governed column is refused before it reaches this line.
774
- """
775
- clean = dict(updates or {})
776
- if not clean:
777
- return
778
- wide = self._tenant_wide_keys()
779
- shared = {k: v for k, v in clean.items() if k in wide}
780
- personal = {k: v for k, v in clean.items() if k not in wide}
781
- if shared:
782
- import core.shared_overlay as shared_overlay
783
- # ⚠ `st=self._st`, NEVER the module default. The two strata must resolve to the SAME
784
- # tenant handle, or a value written by one is invisible to the other and the user's
785
- # edit vanishes the moment they save it (`_ProductTableStore` records the same rule).
786
- shared_overlay.put_cells(self.table_key, pid, shared, st=self._st)
787
- if not personal:
788
- return
789
-
790
- def _patch(ws):
791
- ws['overlays'].setdefault(str(int(pid)), {}).update(personal)
792
-
793
- self._update(username, _patch)
794
-
795
-
796
- def make(table_key, st=None):
797
- return TableStore(table_key, st=st)
 
1
+ """The generic per-user TABLE WORKSPACE store — the persistence half of the table-page factory.
2
+
3
+ One durable store key holds one table OBJECT's per-user Airtable-style state:
4
+
5
+ {username: {'views': {view_id: SavedView},
6
+ 'fields': {field_key: Field}, # notes + custom_ + measure_ strata
7
+ 'overlays': {str(pid): {field_key: value}}}}
8
+
9
+ `make(table_key)` returns the six operations a table page's host loop needs, closed over that
10
+ key. The Customer table's ops (`modules/customer_data.py`, key 'customer_table_workspace') are
11
+ these exact functions — the logic MOVED here 2026-07-27 so that duplicating the Customer table
12
+ pattern to a new object is a registry row + a config, not a copy of the store plumbing
13
+ (owner directive: the table-page factory).
14
+
15
+ A LIST (membership/formula semantics) is deliberately a different store from a VIEW
16
+ (presentation/query state) — see modules/customer_data.py's customer_lists key.
17
+ """
18
+ import core.store as store
19
+
20
+ #: Wave-9 I17 — the SHARED bucket. Views whose permissions make them visible to anyone but
21
+ #: their creator live here instead of in a personal workspace, under a key that cannot collide
22
+ #: with a username (usernames come from core/users.py and are never dunder-wrapped; `is_shared`
23
+ #: guards it anyway). ONE HOME PER VIEW, never both: a view moved to personal is REMOVED from
24
+ #: here, and a view shared is removed from its creator's workspace. Two homes would mean two
25
+ #: divergent copies the moment either was edited.
26
+ SHARED_KEY = '__shared__'
27
+
28
+ #: ⭐⭐ W36-T25 — THE COLUMN SUMMARY IS THE DATABASE'S, NOT ONE ACCOUNT'S.
29
+ #:
30
+ #: Owner item 1, second half, verbatim: *"When i filter Avi on my computer sum works for the Sales
31
+ #: last 365 days column but not when my colleague filter it — The sum of the field is not showing
32
+ #: at the bottom even when i zoom out. It only works on my screen??"* Measured while scouting it:
33
+ #: `save_field` writes the WHOLE field payload into `data[username]`, so a column summary — which
34
+ #: is a statement about the COLUMN, identical for every reader by construction — was stored once
35
+ #: per account. Set it, and the totals row exists for you and for nobody else. `computeAggs` then
36
+ #: paints no totals row at all for the colleague (`showTotals` is false when no field carries an
37
+ #: `agg`), which is exactly "not showing at the bottom".
38
+ #:
39
+ #: ⛔ ONLY `agg` MOVES, AND THE LINE IS NOT ARBITRARY. Width, column order, the note, the display
40
+ #: format and every `custom_`/`measure_` definition stay per user, because each of those is a
41
+ #: statement about how ONE PERSON reads the column. "Sum this column" is a statement about what
42
+ #: the column MEANS, and two accounts disagreeing about it is the defect, not a preference.
43
+ #:
44
+ #: ⚠ IT LIVES IN THE `__shared__` MEMBER OF THIS SAME BUCKET rather than in
45
+ #: `core/shared_overlay.py`'s `<key>__shared` document, and the reason is transactional: a summary
46
+ #: is written by the same `save_field` call that writes the note beside it, and the two must land
47
+ #: or fail together. `__shared__` is already this store's tenant-wide member (shared VIEWS live
48
+ #: there) and is guarded against colliding with a username by `is_shared` and by `core/users.py`'s
49
+ #: never-dunder rule, so there is no second bucket, no second flush and no second failure mode.
50
+ SHARED_FIELD_KEYS = ('agg',)
51
+
52
+
53
+ def _may_see(view, viewer, is_admin=False):
54
+ """Visibility for ONE shared view, fail-closed.
55
+
56
+ 'collaborative' = everyone who can already open the module (the caller has gated that).
57
+ 'users' = the named users, plus the creator, plus admins — an admin who could not
58
+ see a view could not administer it either.
59
+ Anything unrecognised returns False rather than defaulting open: an unreadable permission
60
+ must never widen access ([[aios-permissioning]] — no fail-open defaults).
61
+ """
62
+ if not isinstance(view, dict):
63
+ return False
64
+ if view.get('createdBy') == viewer or is_admin:
65
+ return True
66
+ perms = view.get('permissions') or {}
67
+ edit = perms.get('edit')
68
+ if edit == 'collaborative':
69
+ return True
70
+ if edit == 'users':
71
+ return viewer in set(perms.get('users') or ())
72
+ return False # 'personal', absent, or junk
73
+
74
+
75
+ def _may_edit(view, viewer, is_admin=False):
76
+ """Who may WRITE a shared view. Same set as visibility today — the owner's item asks 'who
77
+ can edit' and lists who 'can have access', i.e. seeing and editing are one grant. Kept as a
78
+ separate function so they can diverge (a future read-only share) without hunting callers."""
79
+ return _may_see(view, viewer, is_admin)
80
+
81
+
82
+ def _may_administer(view, viewer, is_admin=False):
83
+ """Who may change a view's PERMISSIONS, or delete it: the creator or an admin ONLY.
84
+
85
+ Deliberately narrower than _may_edit. If a collaborator could rewrite `permissions` they
86
+ could grant themselves sole ownership of somebody else's view, or quietly widen a
87
+ users-scoped view to everyone — the classic privilege-escalation-by-edit hole.
88
+ """
89
+ if not isinstance(view, dict):
90
+ return False
91
+ return bool(is_admin) or view.get('createdBy') == viewer
92
+
93
+
94
+ def is_shared(view):
95
+ """A view belongs in the shared bucket when its permissions reach beyond its creator."""
96
+ return ((view or {}).get('permissions') or {}).get('edit') in ('collaborative', 'users')
97
+
98
+
99
+ def _shared_fields(data):
100
+ """The tenant-wide field stratum of one workspace document — `{field_key: {'agg': …}}`.
101
+
102
+ ⚠ TOTAL AND FAIL-SOFT: a document written before W36-T25 has no `__shared__` member at all,
103
+ and one written by a future version may have something else in it. Either way this answers an
104
+ empty mapping rather than raising, because the caller is a READ that must still serve the
105
+ workspace — a column with no summary is the state every column was in yesterday.
106
+ """
107
+ shared = (data or {}).get(SHARED_KEY)
108
+ fields = (shared or {}).get('fields') if isinstance(shared, dict) else None
109
+ return fields if isinstance(fields, dict) else {}
110
+
111
+
112
+ def split_shared(payload):
113
+ """`(per_user, shared)` — one field payload divided at the strata boundary.
114
+
115
+ ⭐ W36-T25. `shared` carries only `SHARED_FIELD_KEYS` that are actually SET; `per_user` is the
116
+ payload without them. Split HERE rather than at each write door so the three doors that store
117
+ a field (`save_field`, `duplicate_field`, and the delete that must clear it) cannot come apart
118
+ about where a summary lives — which is the whole reason this module exists rather than being
119
+ copied per topic.
120
+ """
121
+ payload = dict(payload or {})
122
+ shared = {}
123
+ for key in SHARED_FIELD_KEYS:
124
+ value = str(payload.pop(key, '') or '').strip()
125
+ if value:
126
+ shared[key] = value
127
+ return payload, shared
128
+
129
+
130
+ def source_override_is_empty(payload):
131
+ """Does this stored definition of a SOURCE (non-custom) column carry any user state?
132
+
133
+ A cleared note on an immutable source field returns to the canonical schema instead of
134
+ leaving a meaningless override row. Custom fields remain even with an empty note — and so
135
+ does a PRESET field carrying a measure-window override (wave-2 item 8), a DISPLAY-format
136
+ override (wave-5 item 10), and, since W29-T83, a COLUMN SUMMARY.
137
+
138
+ ⛔ EVERY CLAUSE IS A SETTING A USER MADE, and each one omitted is a setting that silently
139
+ stops surviving a session. `agg` was missing: choosing Average on Customer's `Overdue days`
140
+ built an override whose only content was that summary, so this rule threw the whole row away
141
+ on write while the menu went on reading "Summary: Average" from the client's own optimistic
142
+ copy until the next login — a discarded WRITE wearing the face of a failed read
143
+ ([[lost-write-looks-like-failed-read]]). Measured on `bac40c2`; a `ut_*` table, which stores
144
+ its definitions through another door entirely, kept it.
145
+
146
+ ⚠ ONE RULE, TWO CALLERS — here and `grid_events`' store-less fallback. Two copies of a
147
+ discard rule is how one of them keeps a write the other bins ([[one-evaluator-per-question]]).
148
+ ⚠ A CLEARED summary still drops the row, which is the intent: with nothing else set, the
149
+ column goes back to whatever the contract declares for it.
150
+ """
151
+ payload = payload or {}
152
+ if payload.get('custom'):
153
+ return False
154
+ return (not str(payload.get('note') or '').strip()
155
+ and not isinstance(payload.get('measure'), dict)
156
+ and not isinstance(payload.get('format'), dict)
157
+ and not str(payload.get('agg') or '').strip())
158
+
159
+
160
+ def _unique_name(wanted, existing, *, fallback='Untitled', max_len=120):
161
+ """Allocate one human-facing name inside a store.update transaction.
162
+
163
+ Keys/ids remain structural identity. Names compare case-insensitively after collapsing
164
+ whitespace, because those variants are indistinguishable in the UI. This helper belongs
165
+ in the store layer: allocating from a pre-write snapshot lets two concurrent requests both
166
+ choose the same free name before either write lands.
167
+ """
168
+ limit = max(1, int(max_len))
169
+
170
+ def _clean(value):
171
+ return ' '.join(str(value or '').split())
172
+
173
+ base = (_clean(wanted) or _clean(fallback) or 'Untitled')[:limit].rstrip()
174
+ taken = {_clean(value).casefold() for value in existing if _clean(value)}
175
+ if base.casefold() not in taken:
176
+ return base
177
+ index = 2
178
+ while True:
179
+ suffix = f' {index}'
180
+ stem = base[:max(0, limit - len(suffix))].rstrip()
181
+ candidate = f'{stem}{suffix}' if stem else str(index)[-limit:]
182
+ if candidate.casefold() not in taken:
183
+ return candidate
184
+ index += 1
185
+
186
+
187
+ class TableStore:
188
+ """The six store operations for one table object's workspace, closed over its store key.
189
+
190
+ `st` (wave 18, C3-UT) is the STORE HANDLE — anything exposing `get(name)` /
191
+ `update(name, fn, flush=)`. Default = `core.store` (tenant #0, every existing caller,
192
+ zero behaviour change). The API passes the session's `TenantRuntime`, whose accessors
193
+ apply the tenant prefix / repo binding — which is what makes a user table created by a
194
+ Nurilab admin land in Nurilab's store instead of Royal's.
195
+ """
196
+
197
+ def __init__(self, table_key, st=None):
198
+ self.table_key = table_key
199
+ self._st = st if st is not None else store
200
+
201
+ @property
202
+ def st(self):
203
+ """The bound store handle — for SIBLING registries (core/shares) that must read the
204
+ same tenant's buckets this workspace lives in (wave 21, C1)."""
205
+ return self._st
206
+
207
+ def find_view(self, view_id):
208
+ """`(owner_username, view)` for a view living in ANY personal stratum, else None.
209
+
210
+ ⭐ Wave 21 (item 9, C1): the R10 grant registry names bare ids, so projecting a granted
211
+ view means locating the OWNER's record inside this topic's bucket. Personal strata
212
+ only — the `__shared__` bucket has its own read path (`shared_views`), and serving one
213
+ view from two finders is how two copies drift."""
214
+ vid = str(view_id or '').strip()
215
+ if not vid:
216
+ return None
217
+ try:
218
+ data = self._st.get(self.table_key) or {}
219
+ except Exception:
220
+ return None
221
+ for username, ws in data.items():
222
+ if username == SHARED_KEY or not isinstance(ws, dict):
223
+ continue
224
+ v = (ws.get('views') or {}).get(vid)
225
+ if isinstance(v, dict):
226
+ return str(username), dict(v)
227
+ return None
228
+
229
+ def find_folder(self, folder_id):
230
+ """`(owner_username, folder_row, {view_id: view})` for a VIEWS folder living in any
231
+ personal stratum, else None. `find_view`'s sibling, and here for the same reason.
232
+
233
+ ⭐ D-37 (wave 20's R10 remainder, closed 2026-08-05): the grant registry accepts kind
234
+ `folder` and has since wave 20, but only the VIEW kind was ever projected — so "share
235
+ this folder with Karen" recorded a row, listed under Shared with me, and put nothing on
236
+ Karen's screen. Projecting a folder means two lookups the view path does not need: WHO
237
+ owns it, and WHICH views are filed in it. Folder membership lives in the owner's
238
+ `itemFolders` map (item id -> folder id), never on the view record, so the views are
239
+ found by asking that map rather than by reading a list off the folder.
240
+
241
+ Views only (`folders['views']`): the cohort surface has its own store and its own
242
+ sharing question, and answering both here would make one function mean two things.
243
+ """
244
+ fid = str(folder_id or '').strip()
245
+ if not fid:
246
+ return None
247
+ try:
248
+ data = self._st.get(self.table_key) or {}
249
+ except Exception:
250
+ return None
251
+ for username, ws in data.items():
252
+ if username == SHARED_KEY or not isinstance(ws, dict):
253
+ continue
254
+ rows = (ws.get('folders') or {}).get('views') or []
255
+ hit = next((f for f in rows
256
+ if isinstance(f, dict) and str(f.get('id') or '') == fid), None)
257
+ if not hit:
258
+ continue
259
+ # ⚠ `itemFolders` IS KEYED BY SURFACE FIRST (`{'views': {itemId: folderId}, …}`) —
260
+ # reading item ids off the top level finds the surface names instead and matches
261
+ # nothing, so the projection silently returns an EMPTY folder and the feature looks
262
+ # exactly as broken as it was before the fix. Caught by this change's own gate,
263
+ # which is the entire argument for writing one.
264
+ placed = (ws.get('itemFolders') or {}).get('views') or {}
265
+ views = ws.get('views') or {}
266
+ inside = {str(vid): dict(v) for vid, v in views.items()
267
+ if isinstance(v, dict) and str(placed.get(str(vid)) or '') == fid}
268
+ return str(username), dict(hit), inside
269
+ return None
270
+
271
+ # ---------------------------------------------------------------- read
272
+ def workspace(self, username, consume_corrections=True):
273
+ """One user's durable workspace: always the full three-strata shape."""
274
+ try:
275
+ data = self._st.get(self.table_key) or {}
276
+ ws = data.get(username, {}) or {}
277
+ # A collision acknowledgement is protocol state, not part of a field definition.
278
+ # Consume it with the first fresh workspace payload after the correcting write,
279
+ # then splice a bounded copy into that payload only. Keeping it out of `fields`
280
+ # prevents an old request id surviving forever and overriding a later rename.
281
+ corrections = {}
282
+ if consume_corrections and ws.get('fieldCorrections'):
283
+ def _take(current):
284
+ current_ws = current.get(username) or {}
285
+ pending = current_ws.get('fieldCorrections') or {}
286
+ corrections.update({
287
+ str(key)[:80]: dict(value)
288
+ for key, value in pending.items()
289
+ if isinstance(value, dict)
290
+ })
291
+ current_ws.pop('fieldCorrections', None)
292
+ return current
293
+
294
+ data = self._st.update(self.table_key, _take, flush='async')
295
+ ws = (data or {}).get(username, {}) or {}
296
+ except Exception:
297
+ data = {}
298
+ ws = {}
299
+ corrections = {}
300
+ fields = {
301
+ key: dict(value) if isinstance(value, dict) else value
302
+ for key, value in (ws.get('fields') or {}).items()
303
+ }
304
+ # ⭐⭐ W36-T25 — THE TENANT-WIDE COLUMN SUMMARY, MERGED OVER THIS USER'S STRATUM.
305
+ # ⛔ IT MUST BE ABLE TO CREATE AN ENTRY, not only decorate one, and that is the whole
306
+ # reason this is a merge rather than a lookup: the colleague who never touched the column
307
+ # has NO per-user record for it, so a decorate-only pass would have left them with exactly
308
+ # the blank totals row the owner reported. `aios_grid.workspace_wire` reads `meta['agg']`
309
+ # off whatever is here, base column or custom one alike.
310
+ for key, shared in _shared_fields(data).items():
311
+ agg = str((shared or {}).get('agg') or '').strip()
312
+ if not agg:
313
+ continue
314
+ entry = fields.get(key)
315
+ fields[key] = {**entry, 'agg': agg} if isinstance(entry, dict) else {'agg': agg}
316
+ for key, ack in corrections.items():
317
+ field = fields.get(key)
318
+ accepted_label = str(ack.get('label') or '')[:120]
319
+ requested_label = str(ack.get('labelCorrectedFrom') or '')[:120]
320
+ correction_id = str(ack.get('labelCorrectionId') or '')[:180]
321
+ # A newer field write clears/replaces the pending ack in the SAME transaction.
322
+ # The label check is an extra belt against ever attaching a stale ack to a newer
323
+ # definition if a future store implementation weakens that ordering.
324
+ if (isinstance(field, dict) and accepted_label
325
+ and str(field.get('label') or '') == accepted_label
326
+ and requested_label and correction_id):
327
+ field['labelCorrectedFrom'] = requested_label
328
+ field['labelCorrectionId'] = correction_id
329
+ out = {
330
+ 'views': dict(ws.get('views') or {}),
331
+ 'fields': fields,
332
+ 'overlays': dict(ws.get('overlays') or {}),
333
+ # wave-8 I11 (C4): folders over the saved views / cohorts sidebars. A FOURTH
334
+ # stratum rather than a key on each item — see aios_grid.clean_folders for why
335
+ # (a cohort lives in another store, and filing is an organising act, not part of
336
+ # what a view is). Absent for every workspace saved before this wave, which is
337
+ # exactly "no folders yet".
338
+ 'folders': dict(ws.get('folders') or {}),
339
+ 'itemFolders': dict(ws.get('itemFolders') or {}),
340
+ }
341
+ # 2026-07-31 (owner item 3): WHERE THE USER LEFT OFF survives a new browser. The
342
+ # client's localStorage copy wins when present; this is the server's answer for a
343
+ # fresh profile, which used to fall all the way to the system default view.
344
+ if ws.get('activeViewId'):
345
+ out['activeViewId'] = str(ws['activeViewId'])
346
+ # Wave 2026-08-02 (C-LAYOUT): the per-user record-detail field order. A fifth
347
+ # stratum, absent until the user first reorders — exactly "default order".
348
+ if isinstance(ws.get('recordLayout'), dict):
349
+ out['recordLayout'] = dict(ws['recordLayout'])
350
+ return out
351
+
352
+ # ---------------------------------------------------------------- write
353
+ def _update(self, username, change, shared=None):
354
+ """Apply `change(ws)` to this user's stratum, and `shared(shared_fields)` to the
355
+ tenant-wide one, in ONE transaction.
356
+
357
+ ⭐ W36-T25 — `shared` IS A SECOND CALLBACK RATHER THAN A SECOND `update`, and that is the
358
+ whole reason it exists here instead of at the caller. `save_field` writes a note (per
359
+ user) and a column summary (tenant-wide) from ONE payload; two transactions would let the
360
+ summary land while the note did not, and the store's own commit is asynchronous, so the
361
+ window is real rather than theoretical. One `update`, one flush, one failure mode.
362
+ """
363
+ def _up(data):
364
+ ws = data.setdefault(username, {})
365
+ ws.setdefault('views', {})
366
+ ws.setdefault('fields', {})
367
+ ws.setdefault('overlays', {})
368
+ ws.setdefault('folders', {})
369
+ ws.setdefault('itemFolders', {})
370
+ change(ws)
371
+ if shared is not None:
372
+ shared(data.setdefault(SHARED_KEY, {}).setdefault('fields', {}))
373
+ return data
374
+ # flush='async' (wave-7 W3): this is THE hot path — every autosaved filter tweak,
375
+ # column note and typed overlay cell lands here inside the component round-trip, and
376
+ # the historical synchronous hub commit cost seconds per edit. The mutation applies to
377
+ # the in-process cache (read-your-writes for every subsequent render); the hub write
378
+ # coalesces in the background. Registry/auth writes elsewhere stay flush='sync'.
379
+ return self._st.update(self.table_key, _up, flush='async')
380
+
381
+ def rename_choice_values(self, username, change):
382
+ """Apply an arbitrary workspace rewrite (wave 20, item 15 / C-RENAME).
383
+
384
+ ⚠ NAMED FOR ITS ONE CALLER RATHER THAN EXPOSED AS A GENERIC `mutate`, deliberately. A
385
+ public "do anything to the workspace" method is an invitation to put write logic in
386
+ callers instead of here, and every OTHER method on this class exists precisely because
387
+ that logic belongs in one place. Renaming a choice is the one operation that must touch
388
+ three strata AT ONCE — the field's `choices`, the cells in `overlays`, and the views that
389
+ filter or colour by the old value — inside a SINGLE transaction, because a rename that
390
+ updated the cells and not the filters would leave a saved view matching nothing.
391
+
392
+ `change(ws)` receives the whole workspace with every stratum pre-created (see `_update`).
393
+ """
394
+ return self._update(username, change)
395
+
396
+ def save_active_view(self, username, view_id):
397
+ """Remember which view this user last opened (owner item 3, 2026-07-31).
398
+
399
+ Presentation state, not authorisation: the READ side re-validates the id against what
400
+ the caller may actually see, so a stale or foreign id degrades to the default view
401
+ rather than granting anything. Stored per user like every other stratum.
402
+ """
403
+ vid = str(view_id or '').strip()[:120]
404
+ if not vid or username == SHARED_KEY:
405
+ return
406
+
407
+ def _set(ws):
408
+ ws['activeViewId'] = vid
409
+ self._update(username, _set)
410
+
411
+ def save_record_layout(self, username, order):
412
+ """The per-user RECORD-DETAIL field order (wave 2026-08-02, C-LAYOUT).
413
+
414
+ Presentation state for ONE surface — the record modal. Deliberately not view config:
415
+ the owner's ask is per-user, not per-view, and it must never reorder grid columns.
416
+ The event handler validated keys against the live field set; the wire re-validates at
417
+ serve time (aios_grid.workspace_wire), so a deleted field cannot outlive itself here.
418
+ An empty order clears the stratum back to "default order".
419
+ """
420
+ if username == SHARED_KEY:
421
+ return
422
+ clean, seen = [], set()
423
+ for key in (order or [])[:200]:
424
+ key = str(key or '').strip()[:80]
425
+ if key and key not in seen:
426
+ seen.add(key)
427
+ clean.append(key)
428
+
429
+ def _set(ws):
430
+ if clean:
431
+ ws['recordLayout'] = {'order': clean}
432
+ else:
433
+ ws.pop('recordLayout', None)
434
+
435
+ self._update(username, _set)
436
+
437
+ def save_folders(self, username, folders, item_folders):
438
+ """Replace the folder stratum wholesale (wave-8 I11).
439
+
440
+ Wholesale rather than per-folder because the caller has ALREADY validated the complete
441
+ picture through aios_grid.clean_folders / clean_item_folders, and those two are
442
+ interdependent: a placement is only legal while its folder exists, so committing them
443
+ separately would leave a window where a reader sees an item filed into a folder that is
444
+ not there yet. One write, one consistent state.
445
+ """
446
+ def _set(ws):
447
+ ws['folders'] = dict(folders or {})
448
+ ws['itemFolders'] = dict(item_folders or {})
449
+ self._update(username, _set)
450
+
451
+ def save_view_order(self, username, order):
452
+ """⭐ WAVE-27 item 5 (contract C7) — this user's own ORDER for the views rail.
453
+
454
+ Wholesale, like `save_folders` above and for the same reason: the client sends the full
455
+ list it is looking at, not a delta, because a partial order cannot say where an UNNAMED
456
+ view went.
457
+
458
+ ⛔ PER USER, and it belongs in this stratum rather than on the view records themselves.
459
+ `aios_grid`'s own folder note argues it out for placements and every word applies: an
460
+ arrangement is a per-user ORGANISING act, not part of what a view IS — so keeping it out
461
+ of the view config means duplicating, sharing or exporting a view does not drag one
462
+ person's rail position along with it. It also means a SHARED view can sit in a different
463
+ place for each person who can see it, which is the only coherent answer once two people
464
+ share one view.
465
+
466
+ An empty list CLEARS the arrangement (back to server order) rather than storing `[]`.
467
+ """
468
+ def _set(ws):
469
+ clean = []
470
+ seen = set()
471
+ for vid in (order or []):
472
+ vid = str(vid).strip()[:120]
473
+ if vid and vid not in seen:
474
+ seen.add(vid)
475
+ clean.append(vid)
476
+ if clean:
477
+ ws['viewOrder'] = clean
478
+ else:
479
+ ws.pop('viewOrder', None)
480
+ self._update(username, _set)
481
+
482
+ def shared_views(self, viewer, is_admin=False):
483
+ """Every SHARED view this viewer may see, by id (wave-9 I17).
484
+
485
+ Read-only and independent of the viewer's own workspace: the caller merges. Returns
486
+ only what `_may_see` allows, so a caller cannot accidentally render somebody else's
487
+ personal view by forgetting to filter.
488
+ """
489
+ try:
490
+ bucket = ((self._st.get(self.table_key) or {}).get(SHARED_KEY) or {}).get('views') or {}
491
+ except Exception:
492
+ return {}
493
+ return {vid: dict(v) for vid, v in bucket.items()
494
+ if _may_see(v, viewer, is_admin)}
495
+
496
+ def shared_view(self, view_id):
497
+ """One shared view RAW — no visibility filter. For authorisation decisions only: a
498
+ caller must know a view exists and who owns it before it can decide whether the actor
499
+ may touch it. Never hand the result to a renderer without checking `_may_see`."""
500
+ try:
501
+ return ((self._st.get(self.table_key) or {}).get(SHARED_KEY) or {}
502
+ ).get('views', {}).get(str(view_id))
503
+ except Exception:
504
+ return None
505
+
506
+ def save_view(self, username, view, shared=None, reserved_names=(), is_admin=False):
507
+ """Upsert a SavedView into its ONE home — personal workspace or the shared bucket.
508
+
509
+ `shared` defaults to reading the view's own permissions (`is_shared`). Whichever home
510
+ it lands in, the view is REMOVED from the other, so a view can never exist as two
511
+ copies that diverge on the next edit.
512
+
513
+ ⚠ AUTHORISATION IS THE CALLER'S JOB and must happen BEFORE this is called — this layer
514
+ moves data and does not know who is asking. `_cl_handle_one` is the wall.
515
+ """
516
+ view_id = str((view or {}).get('id') or '').strip()
517
+ if not view_id:
518
+ raise ValueError('view id is required')
519
+ if username == SHARED_KEY:
520
+ raise ValueError('reserved username')
521
+ to_shared = is_shared(view) if shared is None else bool(shared)
522
+ requested = dict(view)
523
+ accepted = {}
524
+
525
+ def _up(data):
526
+ # View names are tenant-global: every personal workspace plus the shared bucket.
527
+ # This deliberately includes views the actor cannot see. The only disclosed fact
528
+ # is that a display name is already taken, while the categorical "no duplicate
529
+ # view names" contract remains true when a personal view is later shared.
530
+ names = list(reserved_names or ())
531
+ for workspace in data.values():
532
+ if not isinstance(workspace, dict):
533
+ continue
534
+ names.extend(
535
+ value.get('name')
536
+ for candidate_id, value in (workspace.get('views') or {}).items()
537
+ if candidate_id != view_id and isinstance(value, dict)
538
+ )
539
+ payload = dict(requested)
540
+ payload['name'] = _unique_name(payload.get('name'), names)
541
+ accepted.clear()
542
+ accepted.update(payload)
543
+ if to_shared:
544
+ bucket = data.setdefault(SHARED_KEY, {})
545
+ bucket.setdefault('views', {})[view_id] = payload
546
+ # it may have lived in the creator's workspace before being shared
547
+ owner = data.get(payload.get('createdBy') or username) or {}
548
+ (owner.get('views') or {}).pop(view_id, None)
549
+ else:
550
+ ws = data.setdefault(username, {})
551
+ ws.setdefault('views', {})[view_id] = payload
552
+ (data.get(SHARED_KEY, {}).get('views') or {}).pop(view_id, None)
553
+ return data
554
+
555
+ self._st.update(self.table_key, _up, flush='async')
556
+ return dict(accepted)
557
+
558
+ def delete_view(self, username, view_id):
559
+ """Delete a custom/list view override. The system all-rows view is guarded by caller.
560
+
561
+ Removes from BOTH homes: the caller has already authorised the delete, and leaving a
562
+ stale copy in the other bucket would resurrect the view on the next read.
563
+ """
564
+ vid = str(view_id)
565
+
566
+ def _up(data):
567
+ (data.get(username, {}).get('views') or {}).pop(vid, None)
568
+ (data.get(SHARED_KEY, {}).get('views') or {}).pop(vid, None)
569
+ return data
570
+
571
+ self._st.update(self.table_key, _up, flush='async')
572
+
573
+ def save_field(self, username, field, reserved_names=(), correction_id=None):
574
+ """Persist a column note or a user-created (custom_/measure_) field definition.
575
+
576
+ ⭐⭐ W36-T25 — THE COLUMN SUMMARY GOES TO THE TENANT-WIDE STRATUM, EVERYTHING ELSE STAYS
577
+ PER USER, in ONE transaction. See `SHARED_FIELD_KEYS` above for the owner's report and for
578
+ why the line falls where it does. The returned `accepted` still carries the summary: the
579
+ caller is echoing back the field it just stored, and dropping a key from that echo would
580
+ tell the client its write was refused ([[read-path-cannot-witness-write-path]]).
581
+ """
582
+ key = str((field or {}).get('key') or '').strip()
583
+ if not key:
584
+ raise ValueError('field key is required')
585
+ requested = dict(field)
586
+ accepted = {}
587
+
588
+ def _save(ws):
589
+ names = list(reserved_names or ())
590
+ names.extend(
591
+ value.get('label')
592
+ for candidate_key, value in (ws.get('fields') or {}).items()
593
+ if candidate_key != key and isinstance(value, dict)
594
+ )
595
+ payload = dict(requested)
596
+ payload.pop('labelCorrectedFrom', None)
597
+ payload.pop('labelCorrectionId', None)
598
+ requested_label = ' '.join(
599
+ str(payload.get('label') or 'Untitled').split())[:120].rstrip()
600
+ payload['label'] = _unique_name(requested_label, names)
601
+ corrections = ws.setdefault('fieldCorrections', {})
602
+ corrections.pop(key, None)
603
+ if payload['label'] != requested_label and correction_id:
604
+ corrections[key] = {
605
+ 'label': payload['label'],
606
+ 'labelCorrectedFrom': requested_label,
607
+ 'labelCorrectionId': str(correction_id)[:180],
608
+ }
609
+ if not corrections:
610
+ ws.pop('fieldCorrections', None)
611
+ accepted.clear()
612
+ accepted.update(payload)
613
+ # ⭐⭐ W36-T25 — SPLIT AFTER the label allocation and the correction bookkeeping, so
614
+ # both still see the whole payload, and BEFORE the per-user write.
615
+ mine, shared_now = split_shared(payload)
616
+ shared_write.clear()
617
+ shared_write.update(shared_now)
618
+ # ⚠ EMPTINESS IS JUDGED ON THE PER-USER HALF. A source column whose ONLY state was a
619
+ # summary now has no per-user state at all, and leaving an `{}` override behind is the
620
+ # meaningless row `source_override_is_empty` exists to prevent.
621
+ if source_override_is_empty(mine):
622
+ ws['fields'].pop(key, None)
623
+ else:
624
+ ws['fields'][key] = mine
625
+
626
+ shared_write = {}
627
+
628
+ def _share(shared_fields):
629
+ # ⛔ A CLEARED SUMMARY MUST REMOVE THE ROW, not leave an empty one: `workspace` treats
630
+ # any entry it finds as a live tenant-wide summary, so an `{'agg': ''}` husk would be
631
+ # skipped today and become a resurrection hazard the moment the read grows a second
632
+ # shared key. Absence is the only honest spelling of "nobody set one".
633
+ if shared_write:
634
+ shared_fields[key] = {**(shared_fields.get(key) or {}), **shared_write}
635
+ else:
636
+ shared_fields.pop(key, None)
637
+
638
+ self._update(username, _save, shared=_share)
639
+ return dict(accepted)
640
+
641
+ def duplicate_field(self, username, source_key, new_key, field,
642
+ reserved_names=(), correction_id=None):
643
+ """Clone a user-created field in ONE store transaction (wave-5 item 1): the new
644
+ definition plus — for `custom_` overlay sources only — every stored cell value under
645
+ the source key. One transaction, because a def without its values (or values without a
646
+ def) is exactly the orphan state delete_field exists to prevent, in reverse.
647
+ The caller validated both keys (same created stratum) and stamped the clone's
648
+ createdBy; this layer only moves data."""
649
+ source_key = str(source_key or '').strip()
650
+ new_key = str(new_key or '').strip()
651
+ if not source_key or not new_key or source_key == new_key:
652
+ raise ValueError('duplicate_field needs two distinct keys')
653
+ requested = dict(field)
654
+ accepted = {}
655
+
656
+ def _dup(ws):
657
+ names = list(reserved_names or ())
658
+ names.extend(
659
+ value.get('label')
660
+ for candidate_key, value in (ws.get('fields') or {}).items()
661
+ if candidate_key != new_key and isinstance(value, dict)
662
+ )
663
+ payload = dict(requested)
664
+ payload.pop('labelCorrectedFrom', None)
665
+ payload.pop('labelCorrectionId', None)
666
+ requested_label = ' '.join(
667
+ str(payload.get('label') or 'Untitled').split())[:120].rstrip()
668
+ payload['label'] = _unique_name(requested_label, names)
669
+ corrections = ws.setdefault('fieldCorrections', {})
670
+ corrections.pop(new_key, None)
671
+ if payload['label'] != requested_label and correction_id:
672
+ corrections[new_key] = {
673
+ 'label': payload['label'],
674
+ 'labelCorrectedFrom': requested_label,
675
+ 'labelCorrectionId': str(correction_id)[:180],
676
+ }
677
+ if not corrections:
678
+ ws.pop('fieldCorrections', None)
679
+ accepted.clear()
680
+ accepted.update(payload)
681
+ # ⭐ W36-T25: a CLONE carries the original's summary, and a summary is the database's
682
+ # (see `SHARED_FIELD_KEYS`). Storing it per user here would give the clone a different
683
+ # residency from every other column — one door out of three disagreeing about where a
684
+ # thing lives is how `save_field` and this function drift.
685
+ mine, shared_now = split_shared(payload)
686
+ shared_write.clear()
687
+ shared_write.update(shared_now)
688
+ ws['fields'][new_key] = mine
689
+ if source_key.startswith('custom_'):
690
+ for row in ws['overlays'].values():
691
+ if isinstance(row, dict) and source_key in row:
692
+ row[new_key] = row[source_key]
693
+
694
+ shared_write = {}
695
+
696
+ def _share(shared_fields):
697
+ if shared_write:
698
+ shared_fields[new_key] = {**(shared_fields.get(new_key) or {}), **shared_write}
699
+ else:
700
+ shared_fields.pop(new_key, None)
701
+
702
+ self._update(username, _dup, shared=_share)
703
+ return dict(accepted)
704
+
705
+ def delete_field(self, username, key):
706
+ """Delete a USER-CREATED field definition outright (owner gap closed 2026-07-27).
707
+
708
+ Only the created strata ever reach here (`custom_` overlay fields, `measure_` formula
709
+ columns — the caller enforces the prefix). The stored overlay VALUES for the key are
710
+ scrubbed with it: a deleted column's cells must not linger as orphan data that would
711
+ silently resurface if the key were ever reused. Views referencing the key self-heal on
712
+ their next autosave (an unknown colId is dropped) — the rule every stale key rides.
713
+
714
+ ⭐⭐ W36-T25 — AND THE TENANT-WIDE SUMMARY GOES WITH IT, for exactly the reason the
715
+ paragraph above gives about cells: an orphan `agg` under a deleted key is state nobody can
716
+ see and nobody can clear, and it would attach itself to the next column that happens to
717
+ take the key back. ⚠ This is the ONE stratum a per-user delete may reach across accounts,
718
+ and it is safe because the summary was never this user's to begin with — deleting the
719
+ COLUMN is a tenant-wide act already.
720
+ """
721
+ key = str(key or '').strip()
722
+ if not key:
723
+ return
724
+
725
+ def _drop(ws):
726
+ ws['fields'].pop(key, None)
727
+ for row in ws['overlays'].values():
728
+ if isinstance(row, dict):
729
+ row.pop(key, None)
730
+
731
+ self._update(username, _drop, shared=lambda shared_fields: shared_fields.pop(key, None))
732
+
733
+ def _tenant_wide_keys(self):
734
+ """The columns of THIS table whose values live in the tenant-wide stratum.
735
+
736
+ ⭐ THE CHEAP HALF. A table that shares nothing reads one small per-key file (cache-first
737
+ per process) and answers the empty set, so `patch_overlay` behaves exactly as it did
738
+ before W38-T20 for every database that has no shared column. It is deliberately not
739
+ memoised on the instance: `modules/customer_data.py` and `modules/product_data.py` both
740
+ hold a MODULE-LEVEL `TABLE_OPS`, so a per-instance cache would serve one request's answer
741
+ to the next, and this one decides WHERE a value is written.
742
+ ⚠ LENIENT LIKE EVERY OTHER STRATUM READ, and the failure direction is the safe one: an
743
+ unreachable shared bucket routes the write to the PER-USER stratum, which is the
744
+ pre-ticket behaviour, rather than dropping it.
745
+ """
746
+ try:
747
+ import core.shared_overlay as shared_overlay
748
+ return set(shared_overlay.fields(self.table_key, st=self._st) or ())
749
+ except Exception: # noqa: BLE001
750
+ return set()
751
+
752
+ def patch_overlay(self, username, pid, updates):
753
+ """Patch only the external editable stratum; never writes to the source system.
754
+
755
+ ⭐⭐ W38-T20 / D-423 — A CELL IN A TENANT-WIDE COLUMN GOES TO THE TENANT-WIDE STRATUM,
756
+ AND WITHOUT THIS SPLIT THE EDIT SILENTLY DISAPPEARS. The read path layers the shared
757
+ stratum OVER the per-user one (it has to: that is what makes every reader see the same
758
+ number). This method wrote PER USER. So the sequence was: type a new value, see it accept,
759
+ come back, and read the shared value again — the owner's *"I went back and it all got
760
+ reseted"*, with a successful 200 at every step and nothing in any log.
761
+
762
+ ⛔ IT IS DECIDED BY WHERE THE COLUMN LIVES, NOT BY WHO IS WRITING OR THROUGH WHICH ROUTE.
763
+ `modules/product_data._ProductTableStore` has done exactly this since W30-T36 against its
764
+ CANONICAL list; the only reason it needed a subclass is that its shared columns are
765
+ declared in a contract file. Columns created at runtime cannot be, so the general form
766
+ asks the stratum itself. Both doors (`PATCH /customers/{pid}` and `POST /grid/events`)
767
+ arrive here through `grid_events._tops(ctx)`, which is why the split belongs at the STORE
768
+ and not at either route: intercepting at one leaves the other writing into the shadow.
769
+
770
+ ⛔ NO PERMISSION IS ANSWERED HERE. `shared_overlay`'s header is explicit that it is not a
771
+ wall, and neither is this: whether this session may write this key is settled upstream by
772
+ `EventCtx.hidden_keys`, which `routes_customers._hidden_for` now computes over the MERGED
773
+ contract precisely so a grant-governed column is refused before it reaches this line.
774
+ """
775
+ clean = dict(updates or {})
776
+ if not clean:
777
+ return
778
+ wide = self._tenant_wide_keys()
779
+ shared = {k: v for k, v in clean.items() if k in wide}
780
+ personal = {k: v for k, v in clean.items() if k not in wide}
781
+ if shared:
782
+ import core.shared_overlay as shared_overlay
783
+ # ⚠ `st=self._st`, NEVER the module default. The two strata must resolve to the SAME
784
+ # tenant handle, or a value written by one is invisible to the other and the user's
785
+ # edit vanishes the moment they save it (`_ProductTableStore` records the same rule).
786
+ shared_overlay.put_cells(self.table_key, pid, shared, st=self._st)
787
+ if not personal:
788
+ return
789
+
790
+ def _patch(ws):
791
+ ws['overlays'].setdefault(str(int(pid)), {}).update(personal)
792
+
793
+ self._update(username, _patch)
794
+
795
+
796
+ def make(table_key, st=None):
797
+ return TableStore(table_key, st=st)
platform/core/user_tables.py CHANGED
The diff for this file is too large to render. See raw diff
 
platform/core/users.py CHANGED
@@ -1,532 +1,532 @@
1
- """Per-user accounts for the platform, persisted in the HF Dataset store (users.json).
2
-
3
- Passwords are salted + PBKDF2-HMAC-SHA256 (200k iterations) — never stored or logged in plaintext.
4
- A bootstrap 'admin' account is seeded from APP_PASSWORD so the owner can always log in and create
5
- users; APP_PASSWORD also works as an emergency master for 'admin' if the registry is unreachable.
6
-
7
- Each account carries BU access ('all' or a list of team-ids [5=Fisch, 6=Royal]) which drives
8
- allowed_bus() — the basis for per-Business-Unit permissioning (a Royal-only user never sees Fisch).
9
- """
10
- import os
11
- import hmac
12
- import hashlib
13
- import secrets
14
-
15
- import core.store as store
16
-
17
- BU_LABELS = {5: 'Fisch', 6: 'Royal'}
18
- _ITER = 200_000
19
-
20
- #: Wave 15 C-PERM — the explicit-resolution marker. Mirrors `core.perm_scope.PERMS_VERSION`;
21
- #: kept as a literal here so `users` does not import the permission layer it is read by.
22
- PERMS_VERSION = 1
23
-
24
-
25
- def _hash(pw, salt):
26
- return hashlib.pbkdf2_hmac('sha256', str(pw).encode('utf-8'), bytes.fromhex(salt), _ITER).hex()
27
-
28
-
29
- def _record(pw, name, role, bus, active=True, modules='all', agent=None, email=None,
30
- perms=None, tenant='royal-imports', platform_admin=False):
31
- salt = secrets.token_hex(16)
32
- rec = {'salt': salt, 'hash': _hash(pw, salt), 'name': name, 'role': role,
33
- 'bus': bus, 'active': active, 'modules': modules,
34
- 'agent': agent or None, 'email': email or None,
35
- # Wave 18 (C1-TENANT, R1): the account's COMPANY. Absent == 'royal-imports' on every
36
- # pre-wave record — no migration. Login binds the session to THIS value; the posted
37
- # tenant field can hint but never override it.
38
- 'tenant': str(tenant or 'royal-imports').strip().lower()}
39
- if platform_admin is True:
40
- # Wave 19 (R3): the PLATFORM-operator flag — half of `core.platform_admin`'s double lock
41
- # (the other half is `tenant == 'loopable'`). Written ONLY for True, so every record that
42
- # is not deliberately promoted keeps its pre-wave shape and answers False by absence.
43
- # There is no UI writer and there never should be: it is set by provisioning, on purpose.
44
- rec['platform_admin'] = True
45
- if perms is not None:
46
- # Wave 15 C-PERM. A record written WITH perms is migrated by construction — the marker
47
- # and the block are set together, here, so no writer can create one without the other.
48
- # (`core.perm_scope` reads an unmarked record as legacy, so a block without its marker
49
- # would be silently ignored; a marker without a block would deny everything.)
50
- rec['perms'] = perms
51
- rec['perms_v'] = PERMS_VERSION
52
- return rec
53
-
54
-
55
- def registry():
56
- return store.get('users')
57
-
58
-
59
- def ensure_bootstrap():
60
- """Seed an 'admin' account from APP_PASSWORD ONLY on a truly fresh store (no users file yet).
61
- Idempotent; no-op if the store is unavailable (the app then falls back to the master-password
62
- path in verify()).
63
-
64
- Critically, this NEVER overwrites an existing registry: it seeds only when store.exists('users')
65
- is definitively False. A transient read failure at startup used to return {} and make this
66
- re-seed just {admin} over the real accounts — that is the bug that wiped users on restart."""
67
- if not store.available():
68
- return
69
- if store.exists('users'): # present, or uncertain -> never clobber
70
- return
71
- try:
72
- reg = store.get('users', fresh=True)
73
- except Exception:
74
- return
75
- if reg:
76
- return
77
- master = os.environ.get('APP_PASSWORD', '')
78
- if not master:
79
- return
80
- try:
81
- store.put('users', {'admin': _record(master, 'Administrator', 'admin', 'all')})
82
- except Exception:
83
- pass
84
-
85
-
86
- def _public(username, u):
87
- return {'username': username, 'name': u.get('name', username),
88
- 'role': u.get('role', 'user'), 'bus': u.get('bus', 'all'),
89
- 'modules': u.get('modules', 'all'),
90
- 'agent': u.get('agent'), 'email': u.get('email'),
91
- # Wave 18 (C1-TENANT): the session's tenant binding travels on the projection or it
92
- # does not travel — the same rule the perms block states below.
93
- 'tenant': str(u.get('tenant') or 'royal-imports').strip().lower(),
94
- # Wave 14 C-AVATAR: the profile photo is public-safe by definition (it is served
95
- # to every grid session via the workspace map); without it here the API session's
96
- # user record silently drops it and /me can never show your own photo.
97
- 'avatar': u.get('avatar') or None,
98
- # ⛔ WAVE 15 C-PERM — THE WALL TRAVELS ON THIS PROJECTION OR IT DOES NOT TRAVEL.
99
- # `deps._user_for` builds every API session from `_public()`, so a `perms` block
100
- # dropped here is a restricted account served as an unrestricted one — silently, on
101
- # every route, with nothing to notice. `perms_v` must ride ALONG WITH it and for the
102
- # same reason inverted: the marker without the block denies everything, the block
103
- # without the marker is ignored. Two keys, one fact, never separated.
104
- # `verify_api` asserts a restricted user's SESSION OBJECT carries both, at the mount
105
- # rather than by grep — a projection is exactly the kind of wiring that looks
106
- # present in three files and is absent in the one that runs.
107
- **({'perms': u['perms']} if isinstance(u.get('perms'), dict) else {}),
108
- **({'perms_v': int(u['perms_v'] or 0)} if u.get('perms_v') else {}),
109
- # ⛔ WAVE 19 R3 — THE SAME RULE, ON A NEW FIELD. `deps._user_for` builds every API
110
- # session from this projection, so the platform-admin flag travels here or
111
- # `core.platform_admin.is_platform_admin(session.user)` is blind and the Loopable
112
- # admin plane 403s its own operator. Carried ONLY when the record says True, so a
113
- # session dict for any other account is byte-identical to its pre-wave shape.
114
- # Not a client leak: `routes_auth._public_user` is a whitelist projection and does
115
- # not name this key, so it reaches no browser via /login or /me — the client's copy
116
- # is the separate `platformAdmin` bool on GET /settings, which is derived from this.
117
- **({'platform_admin': True} if u.get('platform_admin') is True else {}),
118
- 'epoch': int(u.get('epoch') or 0)}
119
-
120
-
121
- # ------------------------------------------------------------------ session revocation (X3)
122
- # The API's session cookie is SIGNED AND STATELESS: there is no server-side session table to
123
- # delete from, so "log this user out everywhere" needs a number that lives with the account. The
124
- # cookie carries the epoch it was minted under; bumping the account's epoch makes every
125
- # outstanding cookie for that user fail verification on its next use. Absent == 0, so every
126
- # record written before this wave is valid without a migration.
127
- def epoch(username):
128
- """The current session epoch for `username`. None when there is no such account.
129
-
130
- None is NOT 0. 0 is "this account exists and has never been revoked"; None is "no record" —
131
- which the session verifier must treat as a reason to refuse, not as a default to compare
132
- against. (The APP_PASSWORD emergency-master admin has no record at all; the verifier handles
133
- that case explicitly rather than inventing an epoch for it here.)
134
- """
135
- username = (username or '').strip().lower()
136
- try:
137
- u = (store.get('users') or {}).get(username)
138
- except Exception:
139
- return None
140
- return int((u or {}).get('epoch') or 0) if u else None
141
-
142
-
143
- def bump_epoch(username):
144
- """Revoke every outstanding API session for this account."""
145
- username = (username or '').strip().lower()
146
-
147
- def _set(reg):
148
- u = reg.get(username)
149
- if u:
150
- u['epoch'] = int(u.get('epoch') or 0) + 1
151
- return reg
152
- store.update('users', _set)
153
-
154
-
155
- def verify(username, pw):
156
- """Return a public user dict on success, else None. APP_PASSWORD is an emergency master for the
157
- 'admin' login even if the store is unreachable, so the owner is never locked out."""
158
- username = (username or '').strip().lower()
159
- if not username or not pw:
160
- return None
161
- master = os.environ.get('APP_PASSWORD', '')
162
- try:
163
- # read fresh so accounts created moments ago (UI or out-of-band) are recognised at once
164
- reg = store.get('users', fresh=True)
165
- except Exception:
166
- reg = {}
167
- u = reg.get(username)
168
- # ⛔ KEY FIRST, EMAIL SECOND, AND SINCE WAVE 37 THAT ORDER IS A DECISION RATHER THAN AN ACCIDENT.
169
- # R8 lets a username BE an email address, so a typed string can now match a registry KEY and a
170
- # different account's `email` field at the same time. The key wins, here, by construction: this
171
- # branch is only reached when `reg.get(username)` missed. `routes_admin.py::create_user` refuses
172
- # to CREATE either collision (`username_shadows_email` / `email_shadows_username`) so the
173
- # ambiguity cannot be introduced through the product; this line is what decides it for any
174
- # record that arrived some other way.
175
- if u is None and '@' in username:
176
- # Wave 18 (R1): the login box takes a username OR an email — admin@nurilab.id signs in
177
- # without knowing the slug an admin chose. First case-insensitive email match wins;
178
- # ambiguity is an admin data problem, not a login feature.
179
- for k, r in reg.items():
180
- if isinstance(r, dict) and str(r.get('email') or '').strip().lower() == username:
181
- username, u = k, r
182
- break
183
- if u and u.get('active', True) and hmac.compare_digest(_hash(pw, u['salt']), u['hash']):
184
- return _public(username, u)
185
- # emergency master: admin + APP_PASSWORD always works (covers first run / store outage)
186
- if username == 'admin' and master and hmac.compare_digest(str(pw), master):
187
- # ⚠ CARRY THE RECORD'S CURRENT EPOCH when there is a record to read, so the session cookie
188
- # the API mints from this dict AGREES with the stored account.
189
- #
190
- # This is not what stops the emergency lockout — `deps._user_for`'s master fallback does
191
- # that, and a negative control confirmed the lockout is gone with or without this line.
192
- # What it fixes is subtler and is a SCOPE question: a cookie whose epoch disagrees with the
193
- # record falls through to that master fallback, which hands back a SYNTHETIC identity
194
- # (`bus: 'all'`, `modules: 'all'`). An admin whose record narrows either field would
195
- # therefore be silently WIDENED to consolidated, all-module access for the life of that
196
- # session. Matching the epoch means the record branch wins and the account's real scope
197
- # applies, leaving the master fallback as the true last resort it is meant to be.
198
- # 0 when the store is unreachable, which is the case this branch was written for.
199
- return {'username': 'admin', 'name': 'Administrator', 'role': 'admin', 'bus': 'all',
200
- 'modules': 'all', 'epoch': int((reg.get('admin') or {}).get('epoch') or 0)}
201
- return None
202
-
203
-
204
- def create_user(username, pw, name, role='user', bus='all', modules='all',
205
- agent=None, email=None, tenant=None, platform_admin=None):
206
- """Create — or, from app.py's dialog, OVERWRITE — an account.
207
-
208
- ⛔ X3: OVERWRITING AN ACCOUNT MUST NOT RESURRECT ITS OLD SESSIONS. `_record()` builds a fresh
209
- record with no `epoch` key, i.e. absent == 0. So re-saving an existing username used to reset
210
- the epoch to 0, and every cookie minted before that account's last password rotation started
211
- verifying again — a silent un-revocation. `app.py`'s "Add / update a user" calls this function
212
- for BOTH add and update, so the hole was reachable from the shipped UI.
213
-
214
- Epoch revocation is only ever as strong as the NARROWEST write path that touches the record, so
215
- the carry-and-bump lives here rather than in each caller: an overwrite is at least as
216
- session-invalidating as a password change, and it usually IS one.
217
- """
218
- username = (username or '').strip().lower()
219
- if not username or not pw:
220
- raise ValueError('username and password are required')
221
-
222
- def _add(reg):
223
- prior = reg.get(username) or {}
224
- rec = _record(pw, name or username, role, bus, modules=modules,
225
- agent=agent, email=email,
226
- # An overwrite that names no tenant KEEPS the account's company — a
227
- # rename must never quietly move a user between tenants.
228
- tenant=(tenant or prior.get('tenant') or 'royal-imports'),
229
- # Wave 19 (R3): CARRIED, for the same reason `epoch` is carried below —
230
- # `_record` builds a FRESH record, so re-running the provisioner (it is
231
- # documented as idempotent) or saving an account through this function
232
- # would silently DEMOTE a platform admin and lock the operator out of
233
- # their own plane. None = leave as it was; True/False = set it deliberately.
234
- platform_admin=(prior.get('platform_admin') is True
235
- if platform_admin is None else platform_admin is True))
236
- if prior:
237
- rec['epoch'] = int(prior.get('epoch') or 0) + 1
238
- reg[username] = rec
239
- return reg
240
- store.update('users', _add)
241
-
242
-
243
- def set_password(username, pw):
244
- username = (username or '').strip().lower()
245
-
246
- def _set(reg):
247
- u = reg.get(username)
248
- if u:
249
- u['salt'] = secrets.token_hex(16)
250
- u['hash'] = _hash(pw, u['salt'])
251
- # X3: a password change revokes every outstanding API session for the account. Bumped
252
- # INSIDE the same read-modify-write as the hash so the two can never disagree — a
253
- # separate update() could rotate the password and leave old cookies live if the second
254
- # write failed.
255
- u['epoch'] = int(u.get('epoch') or 0) + 1
256
- return reg
257
- store.update('users', _set)
258
-
259
-
260
- def set_active(username, active):
261
- username = (username or '').strip().lower()
262
-
263
- def _set(reg):
264
- if username in reg:
265
- reg[username]['active'] = bool(active)
266
- # X3: DEACTIVATION must kill live sessions, not just future logins — otherwise a
267
- # disabled account keeps working until its cookie expires. Bumped on reactivation too:
268
- # cheap, and it means a re-enabled account never resurrects a stale cookie.
269
- reg[username]['epoch'] = int(reg[username].get('epoch') or 0) + 1
270
- return reg
271
- store.update('users', _set)
272
-
273
-
274
- def set_platform_admin(username, on):
275
- """Promote/demote a PLATFORM administrator (wave 19, R3) without touching the password.
276
-
277
- The narrow write, deliberately: `create_user` is the destructive path (fresh salt, fresh
278
- hash, bumped epoch) and promoting somebody should not sign them out or rotate a credential.
279
- Cleared by REMOVING the key, so a demoted record goes back to its pre-wave shape rather than
280
- carrying a `False` that reads as "somebody considered this".
281
-
282
- ⚠ This is the flag only. It grants nothing on its own — `core.platform_admin` also demands
283
- the `loopable` tenant, and there is no code path anywhere that moves an account between
284
- tenants, which is what makes the second lock hold.
285
- """
286
- username = (username or '').strip().lower()
287
-
288
- def _set(reg):
289
- u = reg.get(username)
290
- if u:
291
- if on is True:
292
- u['platform_admin'] = True
293
- else:
294
- u.pop('platform_admin', None)
295
- return reg
296
- store.update('users', _set)
297
-
298
-
299
- # ------------------------------------------------------------------ activity stamps (wave 19 R4)
300
- # "When did this account last sign in, and is anyone actually using it?" — the two questions the
301
- # Loopable admin plane exists to answer and that NOTHING in the product could answer before this
302
- # wave (there is no login history, no audit log, no request log anywhere).
303
- #
304
- # ⛔ THIS IS THE FIRST HIGH-FREQUENCY WRITER `users.json` HAS EVER HAD, and that bucket also holds
305
- # every password hash, `active`, `epoch` and the permission blocks. Three rules follow, and the
306
- # second one is a correctness rule, not a performance one:
307
- #
308
- # 1. **In-place mutation of ONE key.** Never `_record()`, never a whole-record replace: a stamp
309
- # that rebuilt the record would reset the salt/hash (locking the user out) or the epoch
310
- # (silently un-revoking every cookie ever minted for them). `set_password`'s docstring
311
- # explains why that class of bug is worth naming out loud.
312
- #
313
- # 2. **SYNCHRONOUS FLUSH — deliberately NOT the async path, and this reverses my first draft.**
314
- # `store.update(flush='async')` rebases on the PROCESS CACHE once a key is `_owned`
315
- # (`core/store.py:284`) and its worker uploads that whole cached blob. For a
316
- # table-workspace key, written by one process, that is exactly right. For `users` it is a
317
- # silent-revert machine: tenant #0's Streamlit host writes the SAME file, so an API process
318
- # holding a cache from an hour ago would, on its next stamp, upload a blob in which a
319
- # password rotation or a deactivation performed in the other host simply never happened.
320
- # A telemetry stamp must not be able to resurrect a disabled account. `flush='sync'` does a
321
- # FRESH strict read inside the store's lock and then uploads, which is the same discipline
322
- # every other `users` writer already uses.
323
- #
324
- # 3. **OFF THE REQUEST THREAD, so rule 2 costs nothing.** A sync commit is a hub round-trip, and
325
- # neither a sign-in nor a random request an hour later should wait for it. Each stamp runs on
326
- # a short-lived daemon thread; `flush_stamps()` is how a test or a shutdown waits for them.
327
- # Everything is fail-silent: a stamp is telemetry and may never turn a good login into a
328
- # failed one — the plane shows an honest "never" instead.
329
- def _now_iso():
330
- import datetime as _dt
331
- return _dt.datetime.now(_dt.timezone.utc).isoformat(timespec='seconds')
332
-
333
-
334
- #: Live stamp threads, so `flush_stamps()` can join them. Bounded by construction — one thread per
335
- #: stamp, and a stamp is at most one per login plus one per account per hour per process.
336
- _STAMPS = []
337
- _STAMPS_LOCK = __import__('threading').Lock()
338
-
339
-
340
- def _stamp(username, fields):
341
- """Merge `fields` into ONE account record, on a background thread, with a fresh read."""
342
- username = (username or '').strip().lower()
343
- if not username or not fields:
344
- return None
345
-
346
- def _set(reg):
347
- u = reg.get(username)
348
- if isinstance(u, dict):
349
- u.update(fields)
350
- return reg
351
-
352
- def _work():
353
- try:
354
- store.update('users', _set) # sync: fresh strict read + blocking upload
355
- except Exception:
356
- pass # a lost stamp is a lost stamp, never an error
357
- import threading
358
- t = threading.Thread(target=_work, daemon=True, name=f'user-stamp:{username}')
359
- with _STAMPS_LOCK:
360
- _STAMPS[:] = [x for x in _STAMPS if x.is_alive()]
361
- _STAMPS.append(t)
362
- t.start()
363
- return t
364
-
365
-
366
- def flush_stamps(timeout=10.0):
367
- """Block until outstanding stamp writes have been applied. For gates and shutdown hooks —
368
- the app never needs it, exactly like `core.store.flush`."""
369
- with _STAMPS_LOCK:
370
- pending = list(_STAMPS)
371
- for t in pending:
372
- t.join(timeout=timeout)
373
- return all(not t.is_alive() for t in pending)
374
-
375
-
376
- def touch_login(username, when=None):
377
- """Stamp `last_login` (ISO-8601, UTC, OFFSET-BEARING) on a SUCCESSFUL login.
378
-
379
- `username` must be the RESOLVED account key, not what the person typed: `verify()` accepts an
380
- email address and resolves it to the registry key, so stamping the typed identifier would
381
- write a stamp onto a key that does not exist and create a phantom account in the registry.
382
-
383
- `last_active` rides along — signing in IS activity, and setting both here means the plane's
384
- two columns agree the moment somebody logs in rather than an hour later.
385
- """
386
- stamp = when or _now_iso()
387
- return _stamp(username, {'last_login': stamp, 'last_active': stamp})
388
-
389
-
390
- def touch_active(username, when=None):
391
- """Stamp `last_active` — "this session did something". Throttled BY THE CALLER (`deps.py`
392
- holds a process-local last-seen map), so this is not a store round-trip per request."""
393
- return _stamp(username, {'last_active': when or _now_iso()})
394
-
395
-
396
- def set_access(username, role=None, bus=None, modules=None, agent=None, email=None,
397
- name=None, perms=None):
398
- """Update access fields. agent/email: pass '' to clear, None to leave unchanged —
399
- the user↔agent link scopes the Customer List page / digests to that agent's book.
400
-
401
- `name` follows the same None-means-unchanged idiom. It is here because it had no setter at all:
402
- a display name could previously only be changed by re-creating the record through
403
- `create_user`, i.e. by also resetting the password (and, before the fix above, the session
404
- epoch). Y4's `PATCH {name?}` needs the narrow write, not the destructive one."""
405
- username = (username or '').strip().lower()
406
-
407
- def _set(reg):
408
- u = reg.get(username)
409
- if u:
410
- if name is not None:
411
- u['name'] = name
412
- if role is not None:
413
- u['role'] = role
414
- if bus is not None:
415
- u['bus'] = bus
416
- if modules is not None:
417
- u['modules'] = modules
418
- if agent is not None:
419
- u['agent'] = agent or None
420
- if email is not None:
421
- u['email'] = email or None
422
- if perms is not None:
423
- # Wave 15 C-PERM. Writing perms MIGRATES the record: the marker goes on in the
424
- # same read-modify-write, so a record can never end up with one and not the
425
- # other (see `_record`). Whole-block replace, matching the PUT route's shape —
426
- # a merge would make "remove this restriction" unexpressible.
427
- u['perms'] = perms
428
- u['perms_v'] = PERMS_VERSION
429
- return reg
430
- store.update('users', _set)
431
-
432
-
433
- def allowed_bus_labels(user):
434
- """BU labels this user may select. 'all' -> All+Fisch+Royal; a single BU -> just that BU (no
435
- 'All', so the other BU is never reachable); multiple -> All + each."""
436
- bus = (user or {}).get('bus', 'all')
437
- if bus == 'all':
438
- return ['All', 'Fisch', 'Royal']
439
- labels = [BU_LABELS[b] for b in bus if b in BU_LABELS]
440
- if not labels:
441
- return ['All', 'Fisch', 'Royal']
442
- return (['All'] + labels) if len(labels) > 1 else labels
443
-
444
-
445
- def assignable_people(tenant=None):
446
- """Display names for `user`-typed overlay columns — the tenant's ACTIVE accounts.
447
-
448
- Moved from app.py (2026-07-31) so both hosts serve the same choices. Resolved on every
449
- call rather than persisted with the column: a snapshot would keep offering people who
450
- have left and never offer people who joined. Deactivated accounts are excluded; a value
451
- already stored on a row is untouched — history should still say who owned something.
452
-
453
- Wave 18 (C1-TENANT): pass `tenant` to scope the choices to ONE company — the user registry
454
- is a global control-plane bucket, and a Nurilab picker offering Royal's staff is a
455
- cross-tenant name leak.
456
-
457
- ⛔⛔ WAVE 33 (W33-T37) — A BLANK `tenant` NOW RETURNS NOTHING. It used to skip the filter and
458
- return EVERY tenant's staff: `if want and …` is structurally fail-OPEN, and the exemption was
459
- written for "None = unscoped (the Streamlit host, tenant #0's process)" — a host DELETED at
460
- EXIT-6. So the sanctioned caller no longer exists, and what was left is a whole-platform
461
- roster one forgotten kwarg away, with no error to notice
462
- ([[gate-must-go-red-not-crash]]'s sibling: a wall that answers instead of refusing).
463
- ⚠ The direction is the safety: this can only ever NARROW. All six live call sites pass
464
- `session.tenant`, which `aios_session.read` refuses to admit blank, so nothing legitimate
465
- changes — and a future caller that forgets gets an empty picker somebody notices instead of a
466
- leak nobody does.
467
- """
468
- return sorted({str(u.get('name') or n) for n, u in _tenant_accounts(tenant)})
469
-
470
-
471
- def set_avatar(username, data_url):
472
- """Set (or clear, with None/'') the user's profile photo — a data URL (wave 14 C-AVATAR,
473
- [[loopable-wave14-split]] item 11). Stored VERBATIM; the API route owns validation (mime +
474
- decoded size) because this value is served back to every grid session. Cleared by removing
475
- the key, so records without a photo keep their pre-wave shape."""
476
- username = (username or '').strip().lower()
477
-
478
- def _set(reg):
479
- u = reg.get(username)
480
- if u:
481
- if data_url:
482
- u['avatar'] = str(data_url)
483
- else:
484
- u.pop('avatar', None)
485
- return reg
486
- store.update('users', _set)
487
-
488
-
489
- def avatar_map(tenant=None):
490
- """{display name -> avatar data URL} for ACTIVE accounts with a photo — the companion of
491
- `assignable_people()`, keyed by the SAME vocabulary: a `user` cell stores the display
492
- name, so the display name is the only join a renderer has. Two active accounts sharing a
493
- display name share one option; the first WITH a photo wins the key rather than a coin
494
- flip deciding whether the option has a face. `tenant` scopes it exactly as
495
- `assignable_people(tenant)` does, and for the same leak — including wave 33's fail-closed
496
- blank, which both take from the ONE resolver below rather than each writing `if want and …`.
497
- """
498
- out = {}
499
- for username, u in sorted(_tenant_accounts(tenant), key=lambda kv: kv[0]):
500
- av = u.get('avatar')
501
- nm = str(u.get('name') or username)
502
- if av and nm not in out:
503
- out[nm] = str(av)
504
- return out
505
-
506
-
507
- def _tenant_accounts(tenant):
508
- """`[(username, record), …]` — the ACTIVE accounts of exactly ONE tenant. FAIL-CLOSED.
509
-
510
- ⛔ THE ONE PLACE THE TENANT FILTER FOR THE USER REGISTRY IS WRITTEN. It was written twice,
511
- identically, in `assignable_people` and `avatar_map`, and both copies were fail-OPEN on a
512
- blank slug in the same way (`if want and …`). Two copies of a wall is two places for one of
513
- them to be fixed [[one-question-two-normalizers]]; `routes_shares._people` is a THIRD copy of
514
- the same predicate and is lane C's to fold in.
515
-
516
- ⚠ `str(u.get('tenant') or 'royal-imports')` is kept on the RECORD side deliberately: a
517
- pre-wave-18 account genuinely has no `tenant` key and IS tenant #0's, and `users._public`
518
- normalises it the same way. What changed is the WANT side — a blank want is now a refusal
519
- rather than a wildcard.
520
- """
521
- want = str(tenant or '').strip().lower()
522
- if not want:
523
- # Not an error and not everything: nobody. See `assignable_people`'s note — the one
524
- # caller this exemption was written for (the Streamlit host) was deleted at EXIT-6.
525
- return []
526
- try:
527
- reg = registry() or {}
528
- except Exception:
529
- return []
530
- return [(n, u) for n, u in reg.items()
531
- if isinstance(u, dict) and u.get('active') is not False
532
- and str(u.get('tenant') or 'royal-imports').strip().lower() == want]
 
1
+ """Per-user accounts for the platform, persisted in the HF Dataset store (users.json).
2
+
3
+ Passwords are salted + PBKDF2-HMAC-SHA256 (200k iterations) — never stored or logged in plaintext.
4
+ A bootstrap 'admin' account is seeded from APP_PASSWORD so the owner can always log in and create
5
+ users; APP_PASSWORD also works as an emergency master for 'admin' if the registry is unreachable.
6
+
7
+ Each account carries BU access ('all' or a list of team-ids [5=Fisch, 6=Royal]) which drives
8
+ allowed_bus() — the basis for per-Business-Unit permissioning (a Royal-only user never sees Fisch).
9
+ """
10
+ import os
11
+ import hmac
12
+ import hashlib
13
+ import secrets
14
+
15
+ import core.store as store
16
+
17
+ BU_LABELS = {5: 'Fisch', 6: 'Royal'}
18
+ _ITER = 200_000
19
+
20
+ #: Wave 15 C-PERM — the explicit-resolution marker. Mirrors `core.perm_scope.PERMS_VERSION`;
21
+ #: kept as a literal here so `users` does not import the permission layer it is read by.
22
+ PERMS_VERSION = 1
23
+
24
+
25
+ def _hash(pw, salt):
26
+ return hashlib.pbkdf2_hmac('sha256', str(pw).encode('utf-8'), bytes.fromhex(salt), _ITER).hex()
27
+
28
+
29
+ def _record(pw, name, role, bus, active=True, modules='all', agent=None, email=None,
30
+ perms=None, tenant='royal-imports', platform_admin=False):
31
+ salt = secrets.token_hex(16)
32
+ rec = {'salt': salt, 'hash': _hash(pw, salt), 'name': name, 'role': role,
33
+ 'bus': bus, 'active': active, 'modules': modules,
34
+ 'agent': agent or None, 'email': email or None,
35
+ # Wave 18 (C1-TENANT, R1): the account's COMPANY. Absent == 'royal-imports' on every
36
+ # pre-wave record — no migration. Login binds the session to THIS value; the posted
37
+ # tenant field can hint but never override it.
38
+ 'tenant': str(tenant or 'royal-imports').strip().lower()}
39
+ if platform_admin is True:
40
+ # Wave 19 (R3): the PLATFORM-operator flag — half of `core.platform_admin`'s double lock
41
+ # (the other half is `tenant == 'loopable'`). Written ONLY for True, so every record that
42
+ # is not deliberately promoted keeps its pre-wave shape and answers False by absence.
43
+ # There is no UI writer and there never should be: it is set by provisioning, on purpose.
44
+ rec['platform_admin'] = True
45
+ if perms is not None:
46
+ # Wave 15 C-PERM. A record written WITH perms is migrated by construction — the marker
47
+ # and the block are set together, here, so no writer can create one without the other.
48
+ # (`core.perm_scope` reads an unmarked record as legacy, so a block without its marker
49
+ # would be silently ignored; a marker without a block would deny everything.)
50
+ rec['perms'] = perms
51
+ rec['perms_v'] = PERMS_VERSION
52
+ return rec
53
+
54
+
55
+ def registry():
56
+ return store.get('users')
57
+
58
+
59
+ def ensure_bootstrap():
60
+ """Seed an 'admin' account from APP_PASSWORD ONLY on a truly fresh store (no users file yet).
61
+ Idempotent; no-op if the store is unavailable (the app then falls back to the master-password
62
+ path in verify()).
63
+
64
+ Critically, this NEVER overwrites an existing registry: it seeds only when store.exists('users')
65
+ is definitively False. A transient read failure at startup used to return {} and make this
66
+ re-seed just {admin} over the real accounts — that is the bug that wiped users on restart."""
67
+ if not store.available():
68
+ return
69
+ if store.exists('users'): # present, or uncertain -> never clobber
70
+ return
71
+ try:
72
+ reg = store.get('users', fresh=True)
73
+ except Exception:
74
+ return
75
+ if reg:
76
+ return
77
+ master = os.environ.get('APP_PASSWORD', '')
78
+ if not master:
79
+ return
80
+ try:
81
+ store.put('users', {'admin': _record(master, 'Administrator', 'admin', 'all')})
82
+ except Exception:
83
+ pass
84
+
85
+
86
+ def _public(username, u):
87
+ return {'username': username, 'name': u.get('name', username),
88
+ 'role': u.get('role', 'user'), 'bus': u.get('bus', 'all'),
89
+ 'modules': u.get('modules', 'all'),
90
+ 'agent': u.get('agent'), 'email': u.get('email'),
91
+ # Wave 18 (C1-TENANT): the session's tenant binding travels on the projection or it
92
+ # does not travel — the same rule the perms block states below.
93
+ 'tenant': str(u.get('tenant') or 'royal-imports').strip().lower(),
94
+ # Wave 14 C-AVATAR: the profile photo is public-safe by definition (it is served
95
+ # to every grid session via the workspace map); without it here the API session's
96
+ # user record silently drops it and /me can never show your own photo.
97
+ 'avatar': u.get('avatar') or None,
98
+ # ⛔ WAVE 15 C-PERM — THE WALL TRAVELS ON THIS PROJECTION OR IT DOES NOT TRAVEL.
99
+ # `deps._user_for` builds every API session from `_public()`, so a `perms` block
100
+ # dropped here is a restricted account served as an unrestricted one — silently, on
101
+ # every route, with nothing to notice. `perms_v` must ride ALONG WITH it and for the
102
+ # same reason inverted: the marker without the block denies everything, the block
103
+ # without the marker is ignored. Two keys, one fact, never separated.
104
+ # `verify_api` asserts a restricted user's SESSION OBJECT carries both, at the mount
105
+ # rather than by grep — a projection is exactly the kind of wiring that looks
106
+ # present in three files and is absent in the one that runs.
107
+ **({'perms': u['perms']} if isinstance(u.get('perms'), dict) else {}),
108
+ **({'perms_v': int(u['perms_v'] or 0)} if u.get('perms_v') else {}),
109
+ # ⛔ WAVE 19 R3 — THE SAME RULE, ON A NEW FIELD. `deps._user_for` builds every API
110
+ # session from this projection, so the platform-admin flag travels here or
111
+ # `core.platform_admin.is_platform_admin(session.user)` is blind and the Loopable
112
+ # admin plane 403s its own operator. Carried ONLY when the record says True, so a
113
+ # session dict for any other account is byte-identical to its pre-wave shape.
114
+ # Not a client leak: `routes_auth._public_user` is a whitelist projection and does
115
+ # not name this key, so it reaches no browser via /login or /me — the client's copy
116
+ # is the separate `platformAdmin` bool on GET /settings, which is derived from this.
117
+ **({'platform_admin': True} if u.get('platform_admin') is True else {}),
118
+ 'epoch': int(u.get('epoch') or 0)}
119
+
120
+
121
+ # ------------------------------------------------------------------ session revocation (X3)
122
+ # The API's session cookie is SIGNED AND STATELESS: there is no server-side session table to
123
+ # delete from, so "log this user out everywhere" needs a number that lives with the account. The
124
+ # cookie carries the epoch it was minted under; bumping the account's epoch makes every
125
+ # outstanding cookie for that user fail verification on its next use. Absent == 0, so every
126
+ # record written before this wave is valid without a migration.
127
+ def epoch(username):
128
+ """The current session epoch for `username`. None when there is no such account.
129
+
130
+ None is NOT 0. 0 is "this account exists and has never been revoked"; None is "no record" —
131
+ which the session verifier must treat as a reason to refuse, not as a default to compare
132
+ against. (The APP_PASSWORD emergency-master admin has no record at all; the verifier handles
133
+ that case explicitly rather than inventing an epoch for it here.)
134
+ """
135
+ username = (username or '').strip().lower()
136
+ try:
137
+ u = (store.get('users') or {}).get(username)
138
+ except Exception:
139
+ return None
140
+ return int((u or {}).get('epoch') or 0) if u else None
141
+
142
+
143
+ def bump_epoch(username):
144
+ """Revoke every outstanding API session for this account."""
145
+ username = (username or '').strip().lower()
146
+
147
+ def _set(reg):
148
+ u = reg.get(username)
149
+ if u:
150
+ u['epoch'] = int(u.get('epoch') or 0) + 1
151
+ return reg
152
+ store.update('users', _set)
153
+
154
+
155
+ def verify(username, pw):
156
+ """Return a public user dict on success, else None. APP_PASSWORD is an emergency master for the
157
+ 'admin' login even if the store is unreachable, so the owner is never locked out."""
158
+ username = (username or '').strip().lower()
159
+ if not username or not pw:
160
+ return None
161
+ master = os.environ.get('APP_PASSWORD', '')
162
+ try:
163
+ # read fresh so accounts created moments ago (UI or out-of-band) are recognised at once
164
+ reg = store.get('users', fresh=True)
165
+ except Exception:
166
+ reg = {}
167
+ u = reg.get(username)
168
+ # ⛔ KEY FIRST, EMAIL SECOND, AND SINCE WAVE 37 THAT ORDER IS A DECISION RATHER THAN AN ACCIDENT.
169
+ # R8 lets a username BE an email address, so a typed string can now match a registry KEY and a
170
+ # different account's `email` field at the same time. The key wins, here, by construction: this
171
+ # branch is only reached when `reg.get(username)` missed. `routes_admin.py::create_user` refuses
172
+ # to CREATE either collision (`username_shadows_email` / `email_shadows_username`) so the
173
+ # ambiguity cannot be introduced through the product; this line is what decides it for any
174
+ # record that arrived some other way.
175
+ if u is None and '@' in username:
176
+ # Wave 18 (R1): the login box takes a username OR an email — admin@nurilab.id signs in
177
+ # without knowing the slug an admin chose. First case-insensitive email match wins;
178
+ # ambiguity is an admin data problem, not a login feature.
179
+ for k, r in reg.items():
180
+ if isinstance(r, dict) and str(r.get('email') or '').strip().lower() == username:
181
+ username, u = k, r
182
+ break
183
+ if u and u.get('active', True) and hmac.compare_digest(_hash(pw, u['salt']), u['hash']):
184
+ return _public(username, u)
185
+ # emergency master: admin + APP_PASSWORD always works (covers first run / store outage)
186
+ if username == 'admin' and master and hmac.compare_digest(str(pw), master):
187
+ # ⚠ CARRY THE RECORD'S CURRENT EPOCH when there is a record to read, so the session cookie
188
+ # the API mints from this dict AGREES with the stored account.
189
+ #
190
+ # This is not what stops the emergency lockout — `deps._user_for`'s master fallback does
191
+ # that, and a negative control confirmed the lockout is gone with or without this line.
192
+ # What it fixes is subtler and is a SCOPE question: a cookie whose epoch disagrees with the
193
+ # record falls through to that master fallback, which hands back a SYNTHETIC identity
194
+ # (`bus: 'all'`, `modules: 'all'`). An admin whose record narrows either field would
195
+ # therefore be silently WIDENED to consolidated, all-module access for the life of that
196
+ # session. Matching the epoch means the record branch wins and the account's real scope
197
+ # applies, leaving the master fallback as the true last resort it is meant to be.
198
+ # 0 when the store is unreachable, which is the case this branch was written for.
199
+ return {'username': 'admin', 'name': 'Administrator', 'role': 'admin', 'bus': 'all',
200
+ 'modules': 'all', 'epoch': int((reg.get('admin') or {}).get('epoch') or 0)}
201
+ return None
202
+
203
+
204
+ def create_user(username, pw, name, role='user', bus='all', modules='all',
205
+ agent=None, email=None, tenant=None, platform_admin=None):
206
+ """Create — or, from app.py's dialog, OVERWRITE — an account.
207
+
208
+ ⛔ X3: OVERWRITING AN ACCOUNT MUST NOT RESURRECT ITS OLD SESSIONS. `_record()` builds a fresh
209
+ record with no `epoch` key, i.e. absent == 0. So re-saving an existing username used to reset
210
+ the epoch to 0, and every cookie minted before that account's last password rotation started
211
+ verifying again — a silent un-revocation. `app.py`'s "Add / update a user" calls this function
212
+ for BOTH add and update, so the hole was reachable from the shipped UI.
213
+
214
+ Epoch revocation is only ever as strong as the NARROWEST write path that touches the record, so
215
+ the carry-and-bump lives here rather than in each caller: an overwrite is at least as
216
+ session-invalidating as a password change, and it usually IS one.
217
+ """
218
+ username = (username or '').strip().lower()
219
+ if not username or not pw:
220
+ raise ValueError('username and password are required')
221
+
222
+ def _add(reg):
223
+ prior = reg.get(username) or {}
224
+ rec = _record(pw, name or username, role, bus, modules=modules,
225
+ agent=agent, email=email,
226
+ # An overwrite that names no tenant KEEPS the account's company — a
227
+ # rename must never quietly move a user between tenants.
228
+ tenant=(tenant or prior.get('tenant') or 'royal-imports'),
229
+ # Wave 19 (R3): CARRIED, for the same reason `epoch` is carried below —
230
+ # `_record` builds a FRESH record, so re-running the provisioner (it is
231
+ # documented as idempotent) or saving an account through this function
232
+ # would silently DEMOTE a platform admin and lock the operator out of
233
+ # their own plane. None = leave as it was; True/False = set it deliberately.
234
+ platform_admin=(prior.get('platform_admin') is True
235
+ if platform_admin is None else platform_admin is True))
236
+ if prior:
237
+ rec['epoch'] = int(prior.get('epoch') or 0) + 1
238
+ reg[username] = rec
239
+ return reg
240
+ store.update('users', _add)
241
+
242
+
243
+ def set_password(username, pw):
244
+ username = (username or '').strip().lower()
245
+
246
+ def _set(reg):
247
+ u = reg.get(username)
248
+ if u:
249
+ u['salt'] = secrets.token_hex(16)
250
+ u['hash'] = _hash(pw, u['salt'])
251
+ # X3: a password change revokes every outstanding API session for the account. Bumped
252
+ # INSIDE the same read-modify-write as the hash so the two can never disagree — a
253
+ # separate update() could rotate the password and leave old cookies live if the second
254
+ # write failed.
255
+ u['epoch'] = int(u.get('epoch') or 0) + 1
256
+ return reg
257
+ store.update('users', _set)
258
+
259
+
260
+ def set_active(username, active):
261
+ username = (username or '').strip().lower()
262
+
263
+ def _set(reg):
264
+ if username in reg:
265
+ reg[username]['active'] = bool(active)
266
+ # X3: DEACTIVATION must kill live sessions, not just future logins — otherwise a
267
+ # disabled account keeps working until its cookie expires. Bumped on reactivation too:
268
+ # cheap, and it means a re-enabled account never resurrects a stale cookie.
269
+ reg[username]['epoch'] = int(reg[username].get('epoch') or 0) + 1
270
+ return reg
271
+ store.update('users', _set)
272
+
273
+
274
+ def set_platform_admin(username, on):
275
+ """Promote/demote a PLATFORM administrator (wave 19, R3) without touching the password.
276
+
277
+ The narrow write, deliberately: `create_user` is the destructive path (fresh salt, fresh
278
+ hash, bumped epoch) and promoting somebody should not sign them out or rotate a credential.
279
+ Cleared by REMOVING the key, so a demoted record goes back to its pre-wave shape rather than
280
+ carrying a `False` that reads as "somebody considered this".
281
+
282
+ ⚠ This is the flag only. It grants nothing on its own — `core.platform_admin` also demands
283
+ the `loopable` tenant, and there is no code path anywhere that moves an account between
284
+ tenants, which is what makes the second lock hold.
285
+ """
286
+ username = (username or '').strip().lower()
287
+
288
+ def _set(reg):
289
+ u = reg.get(username)
290
+ if u:
291
+ if on is True:
292
+ u['platform_admin'] = True
293
+ else:
294
+ u.pop('platform_admin', None)
295
+ return reg
296
+ store.update('users', _set)
297
+
298
+
299
+ # ------------------------------------------------------------------ activity stamps (wave 19 R4)
300
+ # "When did this account last sign in, and is anyone actually using it?" — the two questions the
301
+ # Loopable admin plane exists to answer and that NOTHING in the product could answer before this
302
+ # wave (there is no login history, no audit log, no request log anywhere).
303
+ #
304
+ # ⛔ THIS IS THE FIRST HIGH-FREQUENCY WRITER `users.json` HAS EVER HAD, and that bucket also holds
305
+ # every password hash, `active`, `epoch` and the permission blocks. Three rules follow, and the
306
+ # second one is a correctness rule, not a performance one:
307
+ #
308
+ # 1. **In-place mutation of ONE key.** Never `_record()`, never a whole-record replace: a stamp
309
+ # that rebuilt the record would reset the salt/hash (locking the user out) or the epoch
310
+ # (silently un-revoking every cookie ever minted for them). `set_password`'s docstring
311
+ # explains why that class of bug is worth naming out loud.
312
+ #
313
+ # 2. **SYNCHRONOUS FLUSH — deliberately NOT the async path, and this reverses my first draft.**
314
+ # `store.update(flush='async')` rebases on the PROCESS CACHE once a key is `_owned`
315
+ # (`core/store.py:284`) and its worker uploads that whole cached blob. For a
316
+ # table-workspace key, written by one process, that is exactly right. For `users` it is a
317
+ # silent-revert machine: tenant #0's Streamlit host writes the SAME file, so an API process
318
+ # holding a cache from an hour ago would, on its next stamp, upload a blob in which a
319
+ # password rotation or a deactivation performed in the other host simply never happened.
320
+ # A telemetry stamp must not be able to resurrect a disabled account. `flush='sync'` does a
321
+ # FRESH strict read inside the store's lock and then uploads, which is the same discipline
322
+ # every other `users` writer already uses.
323
+ #
324
+ # 3. **OFF THE REQUEST THREAD, so rule 2 costs nothing.** A sync commit is a hub round-trip, and
325
+ # neither a sign-in nor a random request an hour later should wait for it. Each stamp runs on
326
+ # a short-lived daemon thread; `flush_stamps()` is how a test or a shutdown waits for them.
327
+ # Everything is fail-silent: a stamp is telemetry and may never turn a good login into a
328
+ # failed one — the plane shows an honest "never" instead.
329
+ def _now_iso():
330
+ import datetime as _dt
331
+ return _dt.datetime.now(_dt.timezone.utc).isoformat(timespec='seconds')
332
+
333
+
334
+ #: Live stamp threads, so `flush_stamps()` can join them. Bounded by construction — one thread per
335
+ #: stamp, and a stamp is at most one per login plus one per account per hour per process.
336
+ _STAMPS = []
337
+ _STAMPS_LOCK = __import__('threading').Lock()
338
+
339
+
340
+ def _stamp(username, fields):
341
+ """Merge `fields` into ONE account record, on a background thread, with a fresh read."""
342
+ username = (username or '').strip().lower()
343
+ if not username or not fields:
344
+ return None
345
+
346
+ def _set(reg):
347
+ u = reg.get(username)
348
+ if isinstance(u, dict):
349
+ u.update(fields)
350
+ return reg
351
+
352
+ def _work():
353
+ try:
354
+ store.update('users', _set) # sync: fresh strict read + blocking upload
355
+ except Exception:
356
+ pass # a lost stamp is a lost stamp, never an error
357
+ import threading
358
+ t = threading.Thread(target=_work, daemon=True, name=f'user-stamp:{username}')
359
+ with _STAMPS_LOCK:
360
+ _STAMPS[:] = [x for x in _STAMPS if x.is_alive()]
361
+ _STAMPS.append(t)
362
+ t.start()
363
+ return t
364
+
365
+
366
+ def flush_stamps(timeout=10.0):
367
+ """Block until outstanding stamp writes have been applied. For gates and shutdown hooks —
368
+ the app never needs it, exactly like `core.store.flush`."""
369
+ with _STAMPS_LOCK:
370
+ pending = list(_STAMPS)
371
+ for t in pending:
372
+ t.join(timeout=timeout)
373
+ return all(not t.is_alive() for t in pending)
374
+
375
+
376
+ def touch_login(username, when=None):
377
+ """Stamp `last_login` (ISO-8601, UTC, OFFSET-BEARING) on a SUCCESSFUL login.
378
+
379
+ `username` must be the RESOLVED account key, not what the person typed: `verify()` accepts an
380
+ email address and resolves it to the registry key, so stamping the typed identifier would
381
+ write a stamp onto a key that does not exist and create a phantom account in the registry.
382
+
383
+ `last_active` rides along — signing in IS activity, and setting both here means the plane's
384
+ two columns agree the moment somebody logs in rather than an hour later.
385
+ """
386
+ stamp = when or _now_iso()
387
+ return _stamp(username, {'last_login': stamp, 'last_active': stamp})
388
+
389
+
390
+ def touch_active(username, when=None):
391
+ """Stamp `last_active` — "this session did something". Throttled BY THE CALLER (`deps.py`
392
+ holds a process-local last-seen map), so this is not a store round-trip per request."""
393
+ return _stamp(username, {'last_active': when or _now_iso()})
394
+
395
+
396
+ def set_access(username, role=None, bus=None, modules=None, agent=None, email=None,
397
+ name=None, perms=None):
398
+ """Update access fields. agent/email: pass '' to clear, None to leave unchanged —
399
+ the user↔agent link scopes the Customer List page / digests to that agent's book.
400
+
401
+ `name` follows the same None-means-unchanged idiom. It is here because it had no setter at all:
402
+ a display name could previously only be changed by re-creating the record through
403
+ `create_user`, i.e. by also resetting the password (and, before the fix above, the session
404
+ epoch). Y4's `PATCH {name?}` needs the narrow write, not the destructive one."""
405
+ username = (username or '').strip().lower()
406
+
407
+ def _set(reg):
408
+ u = reg.get(username)
409
+ if u:
410
+ if name is not None:
411
+ u['name'] = name
412
+ if role is not None:
413
+ u['role'] = role
414
+ if bus is not None:
415
+ u['bus'] = bus
416
+ if modules is not None:
417
+ u['modules'] = modules
418
+ if agent is not None:
419
+ u['agent'] = agent or None
420
+ if email is not None:
421
+ u['email'] = email or None
422
+ if perms is not None:
423
+ # Wave 15 C-PERM. Writing perms MIGRATES the record: the marker goes on in the
424
+ # same read-modify-write, so a record can never end up with one and not the
425
+ # other (see `_record`). Whole-block replace, matching the PUT route's shape —
426
+ # a merge would make "remove this restriction" unexpressible.
427
+ u['perms'] = perms
428
+ u['perms_v'] = PERMS_VERSION
429
+ return reg
430
+ store.update('users', _set)
431
+
432
+
433
+ def allowed_bus_labels(user):
434
+ """BU labels this user may select. 'all' -> All+Fisch+Royal; a single BU -> just that BU (no
435
+ 'All', so the other BU is never reachable); multiple -> All + each."""
436
+ bus = (user or {}).get('bus', 'all')
437
+ if bus == 'all':
438
+ return ['All', 'Fisch', 'Royal']
439
+ labels = [BU_LABELS[b] for b in bus if b in BU_LABELS]
440
+ if not labels:
441
+ return ['All', 'Fisch', 'Royal']
442
+ return (['All'] + labels) if len(labels) > 1 else labels
443
+
444
+
445
+ def assignable_people(tenant=None):
446
+ """Display names for `user`-typed overlay columns — the tenant's ACTIVE accounts.
447
+
448
+ Moved from app.py (2026-07-31) so both hosts serve the same choices. Resolved on every
449
+ call rather than persisted with the column: a snapshot would keep offering people who
450
+ have left and never offer people who joined. Deactivated accounts are excluded; a value
451
+ already stored on a row is untouched — history should still say who owned something.
452
+
453
+ Wave 18 (C1-TENANT): pass `tenant` to scope the choices to ONE company — the user registry
454
+ is a global control-plane bucket, and a Nurilab picker offering Royal's staff is a
455
+ cross-tenant name leak.
456
+
457
+ ⛔⛔ WAVE 33 (W33-T37) — A BLANK `tenant` NOW RETURNS NOTHING. It used to skip the filter and
458
+ return EVERY tenant's staff: `if want and …` is structurally fail-OPEN, and the exemption was
459
+ written for "None = unscoped (the Streamlit host, tenant #0's process)" — a host DELETED at
460
+ EXIT-6. So the sanctioned caller no longer exists, and what was left is a whole-platform
461
+ roster one forgotten kwarg away, with no error to notice
462
+ ([[gate-must-go-red-not-crash]]'s sibling: a wall that answers instead of refusing).
463
+ ⚠ The direction is the safety: this can only ever NARROW. All six live call sites pass
464
+ `session.tenant`, which `aios_session.read` refuses to admit blank, so nothing legitimate
465
+ changes — and a future caller that forgets gets an empty picker somebody notices instead of a
466
+ leak nobody does.
467
+ """
468
+ return sorted({str(u.get('name') or n) for n, u in _tenant_accounts(tenant)})
469
+
470
+
471
+ def set_avatar(username, data_url):
472
+ """Set (or clear, with None/'') the user's profile photo — a data URL (wave 14 C-AVATAR,
473
+ [[loopable-wave14-split]] item 11). Stored VERBATIM; the API route owns validation (mime +
474
+ decoded size) because this value is served back to every grid session. Cleared by removing
475
+ the key, so records without a photo keep their pre-wave shape."""
476
+ username = (username or '').strip().lower()
477
+
478
+ def _set(reg):
479
+ u = reg.get(username)
480
+ if u:
481
+ if data_url:
482
+ u['avatar'] = str(data_url)
483
+ else:
484
+ u.pop('avatar', None)
485
+ return reg
486
+ store.update('users', _set)
487
+
488
+
489
+ def avatar_map(tenant=None):
490
+ """{display name -> avatar data URL} for ACTIVE accounts with a photo — the companion of
491
+ `assignable_people()`, keyed by the SAME vocabulary: a `user` cell stores the display
492
+ name, so the display name is the only join a renderer has. Two active accounts sharing a
493
+ display name share one option; the first WITH a photo wins the key rather than a coin
494
+ flip deciding whether the option has a face. `tenant` scopes it exactly as
495
+ `assignable_people(tenant)` does, and for the same leak — including wave 33's fail-closed
496
+ blank, which both take from the ONE resolver below rather than each writing `if want and …`.
497
+ """
498
+ out = {}
499
+ for username, u in sorted(_tenant_accounts(tenant), key=lambda kv: kv[0]):
500
+ av = u.get('avatar')
501
+ nm = str(u.get('name') or username)
502
+ if av and nm not in out:
503
+ out[nm] = str(av)
504
+ return out
505
+
506
+
507
+ def _tenant_accounts(tenant):
508
+ """`[(username, record), …]` — the ACTIVE accounts of exactly ONE tenant. FAIL-CLOSED.
509
+
510
+ ⛔ THE ONE PLACE THE TENANT FILTER FOR THE USER REGISTRY IS WRITTEN. It was written twice,
511
+ identically, in `assignable_people` and `avatar_map`, and both copies were fail-OPEN on a
512
+ blank slug in the same way (`if want and …`). Two copies of a wall is two places for one of
513
+ them to be fixed [[one-question-two-normalizers]]; `routes_shares._people` is a THIRD copy of
514
+ the same predicate and is lane C's to fold in.
515
+
516
+ ⚠ `str(u.get('tenant') or 'royal-imports')` is kept on the RECORD side deliberately: a
517
+ pre-wave-18 account genuinely has no `tenant` key and IS tenant #0's, and `users._public`
518
+ normalises it the same way. What changed is the WANT side — a blank want is now a refusal
519
+ rather than a wildcard.
520
+ """
521
+ want = str(tenant or '').strip().lower()
522
+ if not want:
523
+ # Not an error and not everything: nobody. See `assignable_people`'s note — the one
524
+ # caller this exemption was written for (the Streamlit host) was deleted at EXIT-6.
525
+ return []
526
+ try:
527
+ reg = registry() or {}
528
+ except Exception:
529
+ return []
530
+ return [(n, u) for n, u in reg.items()
531
+ if isinstance(u, dict) and u.get('active') is not False
532
+ and str(u.get('tenant') or 'royal-imports').strip().lower() == want]
platform/harness/datastore.py CHANGED
The diff for this file is too large to render. See raw diff
 
platform/harness/meta_store.py CHANGED
@@ -1,521 +1,521 @@
1
- """meta_store.py — the REST loader for a mirror whose only other loader is XML-RPC (W31-T48).
2
-
3
- python platform/harness/meta_store.py --sync pull into tenant #0's mirror
4
- python platform/harness/meta_store.py --sync --tenant gtmlab
5
- python platform/harness/meta_store.py --status what is in the mirror now
6
-
7
- ⛔ WHY THIS IS A SIBLING AND NOT A ROW IN `datastore.ENTITIES`. `datastore.sync_entity` is
8
- hardwired to XML-RPC — it does `import core.odoo as O` and speaks `search_read` — so no amount of
9
- spec data makes it fetch over HTTPS. What IS reusable is everything *below* the fetch: the
10
- per-tenant file (`path_for`), the process-singleton connection (`connect`), and the
11
- delete-then-insert upsert. This module borrows those and brings its own reader. **The mirror is one
12
- store with two loaders, not two stores.**
13
-
14
- ⭐ THE SHAPE OF THE WHOLE THING, because it is the owner's actual request: *"we just need to pull
15
- the data into our template database for Meta"*, working *"just like Odoo"*. Odoo's path is
16
- `XML-RPC -> DuckDB mirror -> odoo_relational -> ut_odoo_* locked grids`. Meta's is
17
- `Graph -> DuckDB mirror -> meta_relational -> ut_meta_* locked grids`. Same middle, same end, one
18
- different first hop.
19
-
20
- ⚠ EVERY COLUMN NAME BELOW WAS MEASURED, NOT READ OFF A DOC (R8). `proto/meta-entity-fields.json`
21
- carries the run: each level's list is what the API ACCEPTED when asked, on a real ad account.
22
- Ad Account 45 · Campaign 35 · Ad Set 54 · Ad 36 · Creative 58 · Insights 57
23
- ⛔ AND `ACCEPTED` IS NOT `RETURNED`. Graph omits null fields from a response, so a 20-field ask
24
- came back with 14 keys. Building a schema from what came back would drop ~30% of the columns with
25
- nothing going red — which is exactly the "do not drop any column" instruction, broken silently.
26
- The lists below are therefore the ASKED-AND-ACCEPTED set, and a column with no value is NULL.
27
-
28
- ⚠ `owner` IS ABSENT FROM THE AD ACCOUNT LIST AND THAT IS A REPORTED LIMIT, NOT AN OMISSION: it
29
- answers `403 (#200) Requires business_management permission`, which this token does not carry. One
30
- column of 285. R6's second sentence — a limit that cannot be removed gets stated with its cause.
31
-
32
- ⛔ IDS ARE TEXT, ALWAYS. A Meta object id is a 17-digit decimal string; `120273975028650555` is
33
- larger than 2^53, so any float or JS-number path silently corrupts it. Same ruling as `ig_id`
34
- (W26/R3), for the same reason, and it is why every `id` column here is VARCHAR.
35
- """
36
- import argparse
37
- import json
38
- import os
39
- import sys
40
- import time
41
- import urllib.error
42
- import urllib.parse
43
- import urllib.request
44
- from pathlib import Path
45
-
46
- _HERE = Path(__file__).resolve().parent
47
- if str(_HERE.parent) not in sys.path:
48
- sys.path.insert(0, str(_HERE.parent))
49
-
50
- from harness import datastore # noqa: E402
51
-
52
- GRAPH_VERSION = os.environ.get("META_GRAPH_VERSION") or "v21.0"
53
- GRAPH = f"https://graph.facebook.com/{GRAPH_VERSION}"
54
-
55
- #: Numeric columns, by name. Everything else is VARCHAR — including ids (see the header) and
56
- #: including anything Graph returns as a nested object, which is stored as compact JSON text.
57
- _INT = {"impressions", "reach", "clicks", "unique_clicks", "inline_link_clicks",
58
- "inline_post_engagement", "full_view_impressions", "estimated_ad_recallers",
59
- "account_status", "age", "timezone_id", "timezone_offset_hours_utc", "io_number",
60
- "min_daily_budget", "min_campaign_group_spend_cap"}
61
- _DBL = {"spend", "social_spend", "ctr", "unique_ctr", "cpc", "cpm", "cpp", "frequency",
62
- "inline_link_click_ctr", "outbound_clicks_ctr", "amount_spent", "balance", "spend_cap",
63
- "daily_budget", "lifetime_budget", "budget_remaining", "bid_amount", "canvas_avg_view_time",
64
- "daily_min_spend_target", "daily_spend_cap", "lifetime_min_spend_target",
65
- "lifetime_spend_cap", "lifetime_imps"}
66
-
67
- #: ⭐ THE MEASURED CATALOG. `edge` is the connection on the ad account; `None` means the account
68
- #: itself. `parent` names the column that ties a row to its parent, used by nothing here and by
69
- #: the grid links in `meta_relational`.
70
- ACCOUNT_FIELDS = (
71
- "account_id account_status age amount_spent balance business_city business_country_code "
72
- "business_name business_state business_street business_street2 business_zip capabilities "
73
- "created_time currency disable_reason end_advertiser end_advertiser_name funding_source "
74
- "has_migrated_permissions id io_number is_attribution_spec_system_default "
75
- "is_direct_deals_enabled is_notifications_enabled is_personal is_prepay_account "
76
- "is_tax_id_required line_numbers media_agency min_campaign_group_spend_cap min_daily_budget "
77
- "name offsite_pixels_tos_accepted partner spend_cap tax_id tax_id_status tax_id_type "
78
- "timezone_id timezone_name timezone_offset_hours_utc tos_accepted user_tasks user_tos_accepted"
79
- ).split()
80
-
81
- CAMPAIGN_FIELDS = (
82
- "account_id bid_strategy boosted_object_id budget_rebalance_flag budget_remaining buying_type "
83
- "campaign_group_active_time can_create_brand_lift_study can_use_spend_cap configured_status "
84
- "created_time daily_budget effective_status id is_skadnetwork_attribution issues_info "
85
- "last_budget_toggling_time lifetime_budget name objective pacing_type primary_attribution "
86
- "promoted_object smart_promotion_type source_campaign source_campaign_id special_ad_categories "
87
- "special_ad_category special_ad_category_country spend_cap start_time status stop_time "
88
- "topline_id updated_time"
89
- ).split()
90
-
91
- ADSET_FIELDS = (
92
- "account_id adlabels adset_schedule asset_feed_id attribution_spec bid_adjustments bid_amount "
93
- "bid_constraints bid_info bid_strategy billing_event budget_remaining campaign "
94
- "campaign_active_time campaign_attribution campaign_id configured_status created_time "
95
- "creative_sequence daily_budget daily_min_spend_target daily_spend_cap destination_type "
96
- "effective_status end_time frequency_control_specs id instagram_actor_id is_dynamic_creative "
97
- "issues_info learning_stage_info lifetime_budget lifetime_imps lifetime_min_spend_target "
98
- "lifetime_spend_cap multi_optimization_goal_weight name optimization_goal "
99
- "optimization_sub_event pacing_type promoted_object recurring_budget_semantics review_feedback "
100
- "rf_prediction_id source_adset source_adset_id start_time status targeting "
101
- "targeting_optimization_types time_based_ad_rotation_id_blocks "
102
- "time_based_ad_rotation_intervals updated_time use_new_app_click"
103
- ).split()
104
-
105
- AD_FIELDS = (
106
- "account_id ad_active_time ad_review_feedback ad_schedule_end_time ad_schedule_start_time "
107
- "adlabels adset adset_id bid_amount bid_info bid_type campaign campaign_id configured_status "
108
- "conversion_domain created_time creative demolink_hash display_sequence effective_status "
109
- "engagement_audience failed_delivery_checks id issues_info last_updated_by_app_id name "
110
- "preview_shareable_link priority recommendations source_ad source_ad_id status targeting "
111
- "tracking_and_conversion_with_defaults tracking_specs updated_time"
112
- ).split()
113
-
114
- CREATIVE_FIELDS = (
115
- "account_id actor_id adlabels applink_treatment asset_feed_spec authorization_category body "
116
- "branded_content_sponsor_page_id bundle_folder_id call_to_action_type categorization_criteria "
117
- "category_media_source collaborative_ads_lsb_image_bank_id degrees_of_freedom_spec "
118
- "destination_set_id dynamic_ad_voice effective_authorization_category "
119
- "effective_instagram_media_id effective_object_story_id enable_direct_install "
120
- "enable_launch_instant_app id image_crops image_hash image_url instagram_actor_id "
121
- "instagram_permalink_url instagram_story_id instagram_user_id interactive_components_spec "
122
- "link_deep_link_url link_destination_display_url link_og_id link_url "
123
- "messenger_sponsored_message name object_id object_store_url object_story_id object_story_spec "
124
- "object_type object_url place_page_set_id platform_customizations playable_asset_id "
125
- "portrait_customizations product_set_id recommender_settings source_instagram_media_id status "
126
- "template_url template_url_spec thumbnail_id thumbnail_url title url_tags "
127
- "use_page_actor_override video_id"
128
- ).split()
129
-
130
- INSIGHT_FIELDS = (
131
- "account_currency account_id account_name action_values actions ad_id ad_name adset_id "
132
- "adset_name attribution_setting buying_type campaign_id campaign_name canvas_avg_view_time "
133
- "clicks conversion_rate_ranking conversion_values conversions cost_per_action_type "
134
- "cost_per_inline_link_click cost_per_thruplay cost_per_unique_click cpc cpm cpp ctr date_start "
135
- "date_stop engagement_rate_ranking estimated_ad_recallers frequency full_view_impressions "
136
- "impressions inline_link_click_ctr inline_link_clicks inline_post_engagement objective "
137
- "optimization_goal outbound_clicks outbound_clicks_ctr purchase_roas quality_ranking reach "
138
- "social_spend spend unique_clicks unique_ctr unique_outbound_clicks "
139
- "video_avg_time_watched_actions video_p100_watched_actions video_p25_watched_actions "
140
- "video_p50_watched_actions video_p75_watched_actions video_p95_watched_actions "
141
- "video_play_actions video_thruplay_watched_actions website_purchase_roas"
142
- ).split()
143
-
144
- #: table -> (graph edge on the account | None, measured fields, parent column)
145
- SPECS = {
146
- "meta_ad_accounts": (None, ACCOUNT_FIELDS, None),
147
- "meta_campaigns": ("campaigns", CAMPAIGN_FIELDS, "account_id"),
148
- "meta_adsets": ("adsets", ADSET_FIELDS, "campaign_id"),
149
- "meta_ads": ("ads", AD_FIELDS, "adset_id"),
150
- "meta_creatives": ("adcreatives", CREATIVE_FIELDS, "account_id"),
151
- }
152
-
153
- #: The daily Insights grain (R2). One row per (ad, day) — the id is synthesised because Insights
154
- #: has no id of its own, and it must be STABLE so a re-run updates instead of appending.
155
- INSIGHTS_TABLE = "meta_insights_daily"
156
- INSIGHTS_LEVEL = os.environ.get("META_INSIGHTS_LEVEL") or "ad"
157
-
158
- #: ⛔ INSIGHTS IS FETCHED IN TIME SLICES, AND THE REASON IS MEASURED, NOT DEFENSIVE. All 57 fields
159
- #: at ad level over `last_90d` with a 100-row page answers **HTTP 500 "An unknown error occurred"**
160
- #: — Graph's way of saying the synchronous query is too heavy (the async report-run API is the
161
- #: other answer, and it costs a poll loop this does not need). The SAME 57 fields over 7 days at
162
- #: page 25 answer 200. So the window is walked in slices with every column intact:
163
- #: 57 fields · ad level · 7d · limit 25 -> 200, 25 rows
164
- #: 57 fields · account level · 7d -> 200, 7 rows
165
- #: ⚠ Narrowing the FIELD list would also have "fixed" it, and that is the wrong fix twice over —
166
- #: it drops columns R2 requires, and it does so invisibly.
167
- INSIGHTS_DAYS = int(os.environ.get("META_INSIGHTS_DAYS") or 90)
168
- INSIGHTS_SLICE = int(os.environ.get("META_INSIGHTS_SLICE_DAYS") or 7)
169
- INSIGHTS_PAGE = int(os.environ.get("META_INSIGHTS_PAGE") or 25)
170
-
171
-
172
- def _slices(days, size):
173
- """[(since, until)] covering the last `days`, oldest first, in `size`-day windows."""
174
- from datetime import date, timedelta
175
- end = date.today()
176
- start = end - timedelta(days=max(1, days) - 1)
177
- out, cur = [], start
178
- while cur <= end:
179
- stop = min(cur + timedelta(days=max(1, size) - 1), end)
180
- out.append((cur.isoformat(), stop.isoformat()))
181
- cur = stop + timedelta(days=1)
182
- return out
183
-
184
- #: ⚠ A REAL CEILING, AND R6's SECOND SENTENCE APPLIES TO IT. Graph pages at 25-100 rows; this is
185
- #: the number of PAGES a single edge may walk before the loader stops and SAYS it stopped. It is
186
- #: not a row cap on a connected source (R6 forbids that) — it is a runaway guard, and reaching it
187
- #: is reported as a problem, never absorbed.
188
- MAX_PAGES = int(os.environ.get("META_MAX_PAGES") or 200)
189
- PAGE = int(os.environ.get("META_PAGE_SIZE") or 100)
190
-
191
-
192
- class MetaError(RuntimeError):
193
- """A Graph refusal carrying Meta's own words. Safe to print: no token ever reaches it."""
194
-
195
-
196
- def token():
197
- """The Meta token from the environment or gitignored `platform/.env`; "" when absent.
198
-
199
- ⚠ Same resolver `aios-web/api/connectors_meta.py` uses. Duplicated deliberately and minimally:
200
- `platform/` must not import from `aios-web/api/`, which is the layering rule this repo keeps
201
- (`core` never imports up). Fifteen lines is the price of that boundary.
202
- """
203
- tok = os.environ.get("META_ADS_ACCESS_TOKEN") or ""
204
- if tok:
205
- return tok.strip()
206
- env = _HERE.parent / ".env"
207
- if env.exists():
208
- for line in env.read_text(encoding="utf-8", errors="replace").splitlines():
209
- if line.strip().startswith("META_ADS_ACCESS_TOKEN"):
210
- _, _, v = line.partition("=")
211
- return v.strip().strip('"').strip("'")
212
- return ""
213
-
214
-
215
- def _get(path, tok, **params):
216
- params["access_token"] = tok
217
- url = f"{GRAPH}/{path.lstrip('/')}?" + urllib.parse.urlencode(params)
218
- req = urllib.request.Request(url, headers={"User-Agent": "aios-meta-store/1"})
219
- try:
220
- with urllib.request.urlopen(req, timeout=120) as r:
221
- return json.loads(r.read().decode("utf-8", "replace"))
222
- except urllib.error.HTTPError as e:
223
- try:
224
- msg = (json.loads(e.read().decode("utf-8", "replace")).get("error") or {}
225
- ).get("message") or ""
226
- except Exception:
227
- msg = ""
228
- raise MetaError(f"HTTP {e.code} on /{path.lstrip('/')}: {msg[:200]}") from None
229
- except Exception as e:
230
- raise MetaError(f"{type(e).__name__} on /{path.lstrip('/')}") from None
231
-
232
-
233
- #: Graph's own words when a page is too heavy. It arrives as an HTTP **500**, not a 4xx, which is
234
- #: why it cannot be treated as "the server is broken, give up".
235
- _TOO_MUCH = "reduce the amount of data"
236
-
237
- #: ⛔ THE ADS-MANAGEMENT RATE LIMIT, WHICH IS PER AD ACCOUNT AND NOT PER TOKEN. Measured: after a
238
- #: heavy backfill Graph answers **HTTP 400 "There have been too many calls to this ad-account.
239
- #: Wait a bit and try again."** It is a 4xx, so nothing about the status code says "retry" — the
240
- #: MESSAGE is the only signal, which is why it is matched here rather than inferred from a code.
241
- #: ⚠ It persists for minutes, so the backoff is measured in minutes and BOUNDED: after
242
- #: `_RATE_TRIES` waits the loader STOPS and reports what it got, rather than sitting in a retry
243
- #: loop nobody can see. A partial mirror that says it is partial beats a hung sync.
244
- _RATE_LIMITED = "too many calls"
245
- _RATE_TRIES = int(os.environ.get("META_RATE_TRIES") or 3)
246
- _RATE_WAIT = int(os.environ.get("META_RATE_WAIT_S") or 90)
247
-
248
-
249
- def _walk(path, tok, log, **params):
250
- """Every page of an edge, paged by CURSOR under our own parameters.
251
-
252
- ⛔ IT DOES NOT FOLLOW GRAPH'S `paging.next` URL, AND THAT IS THE WHOLE POINT. Measured: the
253
- first call to `/adcreatives` at limit=100 answers **HTTP 500 "Please reduce the amount of data
254
- you're asking for"**, the retry at limit=25 succeeds — and then `next` carries the ORIGINAL
255
- limit=100 and fails again on page 2. A backoff that cannot reach every page is not a backoff.
256
- Re-issuing each page ourselves with `after=<cursor>` keeps the reduced limit for the whole walk.
257
-
258
- ⭐ It shrinks the PAGE, never the FIELD LIST. Dropping columns to make a request fit is the
259
- silent omission R2 forbids, and nothing downstream could see it. Fewer rows per call, always
260
- every column per row.
261
-
262
- -> (rows, hit_page_cap)
263
- """
264
- out, pages = [], 0
265
- limit = int(params.pop("limit", None) or PAGE)
266
- after, waited = None, 0
267
- while True:
268
- call = dict(params, limit=limit)
269
- if after:
270
- call["after"] = after
271
- try:
272
- body = _get(path, tok, **call)
273
- except MetaError as e:
274
- if _TOO_MUCH in str(e) and limit > 5:
275
- limit = max(5, limit // 4)
276
- log(f" page too heavy for /{path} - retrying at limit={limit} "
277
- f"(a payload ceiling, not a row cap; every column is still asked for)")
278
- continue
279
- if _RATE_LIMITED in str(e).lower() and waited < _RATE_TRIES:
280
- waited += 1
281
- log(f" rate-limited on /{path} (per AD ACCOUNT, not per token) - waiting "
282
- f"{_RATE_WAIT}s, attempt {waited}/{_RATE_TRIES}")
283
- time.sleep(_RATE_WAIT)
284
- continue
285
- if _RATE_LIMITED in str(e).lower():
286
- log(f" !! GIVING UP on /{path} after {waited} waits: still rate-limited. "
287
- f"{len(out)} row(s) collected so far are kept. Cause: the ads-management "
288
- f"limit is per ad account and persists for minutes. Fix: re-run "
289
- f"`--sync --only <table>` later; the upsert is idempotent.")
290
- return out, True
291
- raise
292
- out.extend(body.get("data") or [])
293
- pages += 1
294
- after = ((body.get("paging") or {}).get("cursors") or {}).get("after")
295
- has_next = bool((body.get("paging") or {}).get("next")) and bool(after)
296
- if not has_next:
297
- return out, False
298
- if pages >= MAX_PAGES:
299
- log(f" !! STOPPED at MAX_PAGES={MAX_PAGES} on /{path} with more pages left. "
300
- f"Cause: a runaway guard, not a row cap. Fix: raise META_MAX_PAGES, or narrow the "
301
- f"window with META_INSIGHTS_DAYS.")
302
- return out, True
303
-
304
-
305
- def _cell(value):
306
- """One Graph value -> one DuckDB cell. Nested structures become compact JSON TEXT rather than
307
- being dropped: R2 says every field the API returns, and `targeting` is a field."""
308
- if value is None or isinstance(value, (str, int, float, bool)):
309
- return value
310
- return json.dumps(value, separators=(",", ":"), ensure_ascii=False)
311
-
312
-
313
- def _coltype(name, rows=None):
314
- """The DuckDB type for one column — decided by the DATA when there is data, by the name only
315
- as a fallback.
316
-
317
- ⛔ THE NAME LIST WAS WRONG AND ONLY THE API COULD SAY SO. `cost_per_unique_click`,
318
- `cost_per_action_type`, `purchase_roas` and friends READ like money and are **arrays of
319
- action-type objects** on the Insights edge:
320
- [{"action_type":"outbound_click","value":"1.459854"}]
321
- Typed DOUBLE from `_DBL`, the insert died with `Conversion Error: Could not convert string
322
- '[{...}]' to DOUBLE` — after five entity tables had already been written, so the sync looked
323
- like it worked and then blew up on the last table.
324
- ⭐ Same principle the field CATALOG is built on, applied one layer down: **ask the response,
325
- do not assert from a list.** A value that ever arrives as a list or dict is JSON TEXT, because
326
- that is what `_cell` stores; anything else falls back to the measured name hints.
327
- """
328
- if rows:
329
- seen, kinds = 0, set()
330
- for r in rows:
331
- v = r.get(name)
332
- if v is None or v == "":
333
- continue
334
- kinds.add("json" if isinstance(v, (list, dict)) else
335
- "bool" if isinstance(v, bool) else
336
- "int" if isinstance(v, int) else
337
- "float" if isinstance(v, float) else "str")
338
- seen += 1
339
- if seen >= 200:
340
- break
341
- if kinds:
342
- if "json" in kinds or "str" in kinds:
343
- return "VARCHAR" # a JSON blob, or a numeric STRING
344
- if kinds <= {"int", "bool"}:
345
- return "BIGINT" if name in _INT else ("VARCHAR" if "bool" in kinds else "BIGINT")
346
- return "DOUBLE"
347
- if name in _INT:
348
- return "BIGINT"
349
- if name in _DBL:
350
- return "DOUBLE"
351
- return "VARCHAR" # ids included — see the header
352
-
353
-
354
- def _ensure(con, table, fields, rows=None):
355
- """Create or widen the table, with every column typed from `rows` when they are available.
356
-
357
- ⚠ AN EXISTING COLUMN WHOSE TYPE IS NOW WRONG IS REBUILT, NOT PATCHED. DuckDB cannot retype a
358
- column in place, and this table is DERIVED data that can be re-pulled in minutes — so a type
359
- disagreement drops and recreates rather than limping on with a column that refuses every
360
- insert. The alternative is a mirror that is permanently unwritable for one bad guess.
361
- """
362
- want = {f: _coltype(f, rows) for f in fields}
363
- have = {r[1]: str(r[2]).upper() for r in con.execute(f"PRAGMA table_info('{table}')").fetchall()}
364
- if have:
365
- clash = [f for f, ty in want.items() if f in have and have[f] != ty
366
- and not (have[f].startswith("VARCHAR") and ty == "VARCHAR")]
367
- if clash:
368
- con.execute(f"DROP TABLE {table}")
369
- have = {}
370
- if not have:
371
- cols = ", ".join(f"{f} {want[f]}" for f in fields)
372
- con.execute(f"CREATE TABLE IF NOT EXISTS {table} (id VARCHAR PRIMARY KEY, {cols})"
373
- if "id" not in fields else
374
- f"CREATE TABLE IF NOT EXISTS {table} ({cols})")
375
- return
376
- for f in fields:
377
- if f not in have:
378
- con.execute(f"ALTER TABLE {table} ADD COLUMN {f} {want[f]}")
379
-
380
-
381
- def _upsert(con, table, fields, rows):
382
- """Delete-then-insert by id — the same idempotence `datastore._upsert` gives the Odoo half, so
383
- a re-sync updates in place and can never append a second copy of the same object."""
384
- if not rows:
385
- return 0
386
- cols = list(fields)
387
- ids = [str(r.get("id") or "") for r in rows]
388
- q = ",".join("?" for _ in ids)
389
- con.execute(f"DELETE FROM {table} WHERE id IN ({q})", ids)
390
- con.executemany(
391
- f"INSERT INTO {table} ({', '.join(cols)}) VALUES ({', '.join('?' for _ in cols)})",
392
- [[_cell(r.get(c)) for c in cols] for r in rows])
393
- return len(rows)
394
-
395
-
396
- def sync(tenant_key="royal-imports", tok=None, log=print, insights=True, insights_days=None):
397
- """Pull every level into the tenant's mirror. -> a report dict; raises only on a bad token.
398
-
399
- Idempotent: re-running updates rows in place. Safe to call at boot and on the resync loop,
400
- exactly as `odoo_relational.refresh` is.
401
- """
402
- tok = tok or token()
403
- # ⛔ THE WINDOW IS A PARAMETER, NOT AN ENVIRONMENT READ AT CALL TIME — and the difference cost a
404
- # live deploy. `INSIGHTS_DAYS` binds at IMPORT (module scope), so `main._pull_meta`'s
405
- # `os.environ.setdefault("META_INSIGHTS_DAYS", "7")` executed AFTER this module was already
406
- # imported and changed nothing: every boot pulled **90 days**, not 7, which is the slow path
407
- # that trips the per-ad-account rate limit and never finishes. The comment beside that call
408
- # claimed a short window the code could not deliver.
409
- # ⭐ The shape: **a knob read at import cannot be turned by a caller at runtime.** Passing it
410
- # makes the caller's intent effective instead of aspirational; the env var stays the DEFAULT.
411
- days = int(insights_days or INSIGHTS_DAYS)
412
- report = {"tenant": tenant_key, "tables": {}, "problems": [], "accounts": []}
413
- if not tok:
414
- report["problems"].append(
415
- "META_ADS_ACCESS_TOKEN is not set in the environment or in platform/.env, so nothing "
416
- "was pulled. This is a missing CREDENTIAL, not a missing capability.")
417
- return report
418
-
419
- path = datastore.path_for(tenant_key)
420
- if Path(datastore.DB_PATH) != Path(path):
421
- datastore.use_path(path)
422
- con = datastore.connect()
423
- log(f" mirror: {Path(path).name}")
424
-
425
- accts, _ = _walk("me/adaccounts", tok, log, fields="id,name", limit=PAGE)
426
- report["accounts"] = [a.get("id") for a in accts]
427
- if not accts:
428
- report["problems"].append("the token reaches no ad accounts")
429
- return report
430
-
431
- # ── the five entity levels ────────────────────────────────────────────────────────────────
432
- for table, (edge, fields, _parent) in SPECS.items():
433
- rows, capped = [], False
434
- for acct in accts:
435
- aid = acct["id"]
436
- if edge is None:
437
- rows.append(_get(aid, tok, fields=",".join(fields)))
438
- else:
439
- got, hit = _walk(f"{aid}/{edge}", tok, log, fields=",".join(fields), limit=PAGE)
440
- rows.extend(got)
441
- capped = capped or hit
442
- _ensure(con, table, fields, rows)
443
- n = _upsert(con, table, fields, rows)
444
- total = con.execute(f"SELECT count(*) FROM {table}").fetchone()[0]
445
- report["tables"][table] = {"pulled": n, "in_mirror": total, "capped": capped}
446
- log(f" {table:<20} pulled {n:>6} mirror total {total:>6}"
447
- + (" !! PAGE CAP HIT" if capped else ""))
448
- if capped:
449
- report["problems"].append(f"{table}: stopped at MAX_PAGES={MAX_PAGES}")
450
-
451
- # ── the daily Insights grain ──────────────────────────────────────────────────────────────
452
- if insights:
453
- rows, capped = [], False
454
- for acct in accts:
455
- for since, until in _slices(days, INSIGHTS_SLICE):
456
- got, hit = _walk(f"{acct['id']}/insights", tok, log,
457
- fields=",".join(INSIGHT_FIELDS), level=INSIGHTS_LEVEL,
458
- time_increment="1", limit=INSIGHTS_PAGE,
459
- time_range=json.dumps({"since": since, "until": until}))
460
- rows.extend(got)
461
- capped = capped or hit
462
- # ⛔ Insights rows carry no id. The key must be STABLE across runs or every re-sync
463
- # appends a second copy of the same day — so it is composed from the grain itself.
464
- for r in rows:
465
- r["id"] = ":".join(str(r.get(k) or "") for k in
466
- (f"{INSIGHTS_LEVEL}_id", "date_start", "date_stop"))
467
- fields = ["id"] + INSIGHT_FIELDS
468
- _ensure(con, INSIGHTS_TABLE, fields, rows)
469
- n = _upsert(con, INSIGHTS_TABLE, fields, rows)
470
- total = con.execute(f"SELECT count(*) FROM {INSIGHTS_TABLE}").fetchone()[0]
471
- report["tables"][INSIGHTS_TABLE] = {"pulled": n, "in_mirror": total, "capped": capped}
472
- log(f" {INSIGHTS_TABLE:<20} pulled {n:>6} mirror total {total:>6}"
473
- + (" !! PAGE CAP HIT" if capped else ""))
474
-
475
- con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
476
- ["meta", "done", 0, f"{days}d/{INSIGHTS_SLICE}d@{INSIGHTS_LEVEL}",
477
- sum(t["in_mirror"] for t in report["tables"].values()),
478
- time.strftime("%Y-%m-%d %H:%M:%S")])
479
- return report
480
-
481
-
482
- def status(tenant_key="royal-imports"):
483
- """What is in the mirror right now, per table. Never fetches."""
484
- path = datastore.path_for(tenant_key)
485
- if Path(datastore.DB_PATH) != Path(path):
486
- datastore.use_path(path)
487
- out = {}
488
- con = datastore.connect()
489
- for table in list(SPECS) + [INSIGHTS_TABLE]:
490
- try:
491
- out[table] = con.execute(f"SELECT count(*) FROM {table}").fetchone()[0]
492
- except Exception:
493
- out[table] = None # table absent = never synced
494
- return out
495
-
496
-
497
- def main(argv=None):
498
- ap = argparse.ArgumentParser(description="Pull Meta Ads into the tenant's DuckDB mirror.")
499
- ap.add_argument("--sync", action="store_true")
500
- ap.add_argument("--status", action="store_true")
501
- ap.add_argument("--tenant", default="royal-imports")
502
- ap.add_argument("--no-insights", action="store_true")
503
- ap.add_argument("--insights-days", type=int, default=None,
504
- help="override the Insights window for THIS run (default INSIGHTS_DAYS); the env var is the default, this is the caller's say")
505
- a = ap.parse_args(argv)
506
- if a.status:
507
- for k, v in status(a.tenant).items():
508
- print(f" {k:<20} {'(never synced)' if v is None else v}")
509
- return 0
510
- if not a.sync:
511
- ap.print_help()
512
- return 2
513
- rep = sync(a.tenant, insights=not a.no_insights, insights_days=a.insights_days)
514
- for p in rep["problems"]:
515
- print(" PROBLEM:", p)
516
- print(f" accounts: {len(rep['accounts'])} tables: {len(rep['tables'])}")
517
- return 1 if rep["problems"] else 0
518
-
519
-
520
- if __name__ == "__main__":
521
- sys.exit(main())
 
1
+ """meta_store.py — the REST loader for a mirror whose only other loader is XML-RPC (W31-T48).
2
+
3
+ python platform/harness/meta_store.py --sync pull into tenant #0's mirror
4
+ python platform/harness/meta_store.py --sync --tenant gtmlab
5
+ python platform/harness/meta_store.py --status what is in the mirror now
6
+
7
+ ⛔ WHY THIS IS A SIBLING AND NOT A ROW IN `datastore.ENTITIES`. `datastore.sync_entity` is
8
+ hardwired to XML-RPC — it does `import core.odoo as O` and speaks `search_read` — so no amount of
9
+ spec data makes it fetch over HTTPS. What IS reusable is everything *below* the fetch: the
10
+ per-tenant file (`path_for`), the process-singleton connection (`connect`), and the
11
+ delete-then-insert upsert. This module borrows those and brings its own reader. **The mirror is one
12
+ store with two loaders, not two stores.**
13
+
14
+ ⭐ THE SHAPE OF THE WHOLE THING, because it is the owner's actual request: *"we just need to pull
15
+ the data into our template database for Meta"*, working *"just like Odoo"*. Odoo's path is
16
+ `XML-RPC -> DuckDB mirror -> odoo_relational -> ut_odoo_* locked grids`. Meta's is
17
+ `Graph -> DuckDB mirror -> meta_relational -> ut_meta_* locked grids`. Same middle, same end, one
18
+ different first hop.
19
+
20
+ ⚠ EVERY COLUMN NAME BELOW WAS MEASURED, NOT READ OFF A DOC (R8). `proto/meta-entity-fields.json`
21
+ carries the run: each level's list is what the API ACCEPTED when asked, on a real ad account.
22
+ Ad Account 45 · Campaign 35 · Ad Set 54 · Ad 36 · Creative 58 · Insights 57
23
+ ⛔ AND `ACCEPTED` IS NOT `RETURNED`. Graph omits null fields from a response, so a 20-field ask
24
+ came back with 14 keys. Building a schema from what came back would drop ~30% of the columns with
25
+ nothing going red — which is exactly the "do not drop any column" instruction, broken silently.
26
+ The lists below are therefore the ASKED-AND-ACCEPTED set, and a column with no value is NULL.
27
+
28
+ ⚠ `owner` IS ABSENT FROM THE AD ACCOUNT LIST AND THAT IS A REPORTED LIMIT, NOT AN OMISSION: it
29
+ answers `403 (#200) Requires business_management permission`, which this token does not carry. One
30
+ column of 285. R6's second sentence — a limit that cannot be removed gets stated with its cause.
31
+
32
+ ⛔ IDS ARE TEXT, ALWAYS. A Meta object id is a 17-digit decimal string; `120273975028650555` is
33
+ larger than 2^53, so any float or JS-number path silently corrupts it. Same ruling as `ig_id`
34
+ (W26/R3), for the same reason, and it is why every `id` column here is VARCHAR.
35
+ """
36
+ import argparse
37
+ import json
38
+ import os
39
+ import sys
40
+ import time
41
+ import urllib.error
42
+ import urllib.parse
43
+ import urllib.request
44
+ from pathlib import Path
45
+
46
+ _HERE = Path(__file__).resolve().parent
47
+ if str(_HERE.parent) not in sys.path:
48
+ sys.path.insert(0, str(_HERE.parent))
49
+
50
+ from harness import datastore # noqa: E402
51
+
52
+ GRAPH_VERSION = os.environ.get("META_GRAPH_VERSION") or "v21.0"
53
+ GRAPH = f"https://graph.facebook.com/{GRAPH_VERSION}"
54
+
55
+ #: Numeric columns, by name. Everything else is VARCHAR — including ids (see the header) and
56
+ #: including anything Graph returns as a nested object, which is stored as compact JSON text.
57
+ _INT = {"impressions", "reach", "clicks", "unique_clicks", "inline_link_clicks",
58
+ "inline_post_engagement", "full_view_impressions", "estimated_ad_recallers",
59
+ "account_status", "age", "timezone_id", "timezone_offset_hours_utc", "io_number",
60
+ "min_daily_budget", "min_campaign_group_spend_cap"}
61
+ _DBL = {"spend", "social_spend", "ctr", "unique_ctr", "cpc", "cpm", "cpp", "frequency",
62
+ "inline_link_click_ctr", "outbound_clicks_ctr", "amount_spent", "balance", "spend_cap",
63
+ "daily_budget", "lifetime_budget", "budget_remaining", "bid_amount", "canvas_avg_view_time",
64
+ "daily_min_spend_target", "daily_spend_cap", "lifetime_min_spend_target",
65
+ "lifetime_spend_cap", "lifetime_imps"}
66
+
67
+ #: ⭐ THE MEASURED CATALOG. `edge` is the connection on the ad account; `None` means the account
68
+ #: itself. `parent` names the column that ties a row to its parent, used by nothing here and by
69
+ #: the grid links in `meta_relational`.
70
+ ACCOUNT_FIELDS = (
71
+ "account_id account_status age amount_spent balance business_city business_country_code "
72
+ "business_name business_state business_street business_street2 business_zip capabilities "
73
+ "created_time currency disable_reason end_advertiser end_advertiser_name funding_source "
74
+ "has_migrated_permissions id io_number is_attribution_spec_system_default "
75
+ "is_direct_deals_enabled is_notifications_enabled is_personal is_prepay_account "
76
+ "is_tax_id_required line_numbers media_agency min_campaign_group_spend_cap min_daily_budget "
77
+ "name offsite_pixels_tos_accepted partner spend_cap tax_id tax_id_status tax_id_type "
78
+ "timezone_id timezone_name timezone_offset_hours_utc tos_accepted user_tasks user_tos_accepted"
79
+ ).split()
80
+
81
+ CAMPAIGN_FIELDS = (
82
+ "account_id bid_strategy boosted_object_id budget_rebalance_flag budget_remaining buying_type "
83
+ "campaign_group_active_time can_create_brand_lift_study can_use_spend_cap configured_status "
84
+ "created_time daily_budget effective_status id is_skadnetwork_attribution issues_info "
85
+ "last_budget_toggling_time lifetime_budget name objective pacing_type primary_attribution "
86
+ "promoted_object smart_promotion_type source_campaign source_campaign_id special_ad_categories "
87
+ "special_ad_category special_ad_category_country spend_cap start_time status stop_time "
88
+ "topline_id updated_time"
89
+ ).split()
90
+
91
+ ADSET_FIELDS = (
92
+ "account_id adlabels adset_schedule asset_feed_id attribution_spec bid_adjustments bid_amount "
93
+ "bid_constraints bid_info bid_strategy billing_event budget_remaining campaign "
94
+ "campaign_active_time campaign_attribution campaign_id configured_status created_time "
95
+ "creative_sequence daily_budget daily_min_spend_target daily_spend_cap destination_type "
96
+ "effective_status end_time frequency_control_specs id instagram_actor_id is_dynamic_creative "
97
+ "issues_info learning_stage_info lifetime_budget lifetime_imps lifetime_min_spend_target "
98
+ "lifetime_spend_cap multi_optimization_goal_weight name optimization_goal "
99
+ "optimization_sub_event pacing_type promoted_object recurring_budget_semantics review_feedback "
100
+ "rf_prediction_id source_adset source_adset_id start_time status targeting "
101
+ "targeting_optimization_types time_based_ad_rotation_id_blocks "
102
+ "time_based_ad_rotation_intervals updated_time use_new_app_click"
103
+ ).split()
104
+
105
+ AD_FIELDS = (
106
+ "account_id ad_active_time ad_review_feedback ad_schedule_end_time ad_schedule_start_time "
107
+ "adlabels adset adset_id bid_amount bid_info bid_type campaign campaign_id configured_status "
108
+ "conversion_domain created_time creative demolink_hash display_sequence effective_status "
109
+ "engagement_audience failed_delivery_checks id issues_info last_updated_by_app_id name "
110
+ "preview_shareable_link priority recommendations source_ad source_ad_id status targeting "
111
+ "tracking_and_conversion_with_defaults tracking_specs updated_time"
112
+ ).split()
113
+
114
+ CREATIVE_FIELDS = (
115
+ "account_id actor_id adlabels applink_treatment asset_feed_spec authorization_category body "
116
+ "branded_content_sponsor_page_id bundle_folder_id call_to_action_type categorization_criteria "
117
+ "category_media_source collaborative_ads_lsb_image_bank_id degrees_of_freedom_spec "
118
+ "destination_set_id dynamic_ad_voice effective_authorization_category "
119
+ "effective_instagram_media_id effective_object_story_id enable_direct_install "
120
+ "enable_launch_instant_app id image_crops image_hash image_url instagram_actor_id "
121
+ "instagram_permalink_url instagram_story_id instagram_user_id interactive_components_spec "
122
+ "link_deep_link_url link_destination_display_url link_og_id link_url "
123
+ "messenger_sponsored_message name object_id object_store_url object_story_id object_story_spec "
124
+ "object_type object_url place_page_set_id platform_customizations playable_asset_id "
125
+ "portrait_customizations product_set_id recommender_settings source_instagram_media_id status "
126
+ "template_url template_url_spec thumbnail_id thumbnail_url title url_tags "
127
+ "use_page_actor_override video_id"
128
+ ).split()
129
+
130
+ INSIGHT_FIELDS = (
131
+ "account_currency account_id account_name action_values actions ad_id ad_name adset_id "
132
+ "adset_name attribution_setting buying_type campaign_id campaign_name canvas_avg_view_time "
133
+ "clicks conversion_rate_ranking conversion_values conversions cost_per_action_type "
134
+ "cost_per_inline_link_click cost_per_thruplay cost_per_unique_click cpc cpm cpp ctr date_start "
135
+ "date_stop engagement_rate_ranking estimated_ad_recallers frequency full_view_impressions "
136
+ "impressions inline_link_click_ctr inline_link_clicks inline_post_engagement objective "
137
+ "optimization_goal outbound_clicks outbound_clicks_ctr purchase_roas quality_ranking reach "
138
+ "social_spend spend unique_clicks unique_ctr unique_outbound_clicks "
139
+ "video_avg_time_watched_actions video_p100_watched_actions video_p25_watched_actions "
140
+ "video_p50_watched_actions video_p75_watched_actions video_p95_watched_actions "
141
+ "video_play_actions video_thruplay_watched_actions website_purchase_roas"
142
+ ).split()
143
+
144
+ #: table -> (graph edge on the account | None, measured fields, parent column)
145
+ SPECS = {
146
+ "meta_ad_accounts": (None, ACCOUNT_FIELDS, None),
147
+ "meta_campaigns": ("campaigns", CAMPAIGN_FIELDS, "account_id"),
148
+ "meta_adsets": ("adsets", ADSET_FIELDS, "campaign_id"),
149
+ "meta_ads": ("ads", AD_FIELDS, "adset_id"),
150
+ "meta_creatives": ("adcreatives", CREATIVE_FIELDS, "account_id"),
151
+ }
152
+
153
+ #: The daily Insights grain (R2). One row per (ad, day) — the id is synthesised because Insights
154
+ #: has no id of its own, and it must be STABLE so a re-run updates instead of appending.
155
+ INSIGHTS_TABLE = "meta_insights_daily"
156
+ INSIGHTS_LEVEL = os.environ.get("META_INSIGHTS_LEVEL") or "ad"
157
+
158
+ #: ⛔ INSIGHTS IS FETCHED IN TIME SLICES, AND THE REASON IS MEASURED, NOT DEFENSIVE. All 57 fields
159
+ #: at ad level over `last_90d` with a 100-row page answers **HTTP 500 "An unknown error occurred"**
160
+ #: — Graph's way of saying the synchronous query is too heavy (the async report-run API is the
161
+ #: other answer, and it costs a poll loop this does not need). The SAME 57 fields over 7 days at
162
+ #: page 25 answer 200. So the window is walked in slices with every column intact:
163
+ #: 57 fields · ad level · 7d · limit 25 -> 200, 25 rows
164
+ #: 57 fields · account level · 7d -> 200, 7 rows
165
+ #: ⚠ Narrowing the FIELD list would also have "fixed" it, and that is the wrong fix twice over —
166
+ #: it drops columns R2 requires, and it does so invisibly.
167
+ INSIGHTS_DAYS = int(os.environ.get("META_INSIGHTS_DAYS") or 90)
168
+ INSIGHTS_SLICE = int(os.environ.get("META_INSIGHTS_SLICE_DAYS") or 7)
169
+ INSIGHTS_PAGE = int(os.environ.get("META_INSIGHTS_PAGE") or 25)
170
+
171
+
172
+ def _slices(days, size):
173
+ """[(since, until)] covering the last `days`, oldest first, in `size`-day windows."""
174
+ from datetime import date, timedelta
175
+ end = date.today()
176
+ start = end - timedelta(days=max(1, days) - 1)
177
+ out, cur = [], start
178
+ while cur <= end:
179
+ stop = min(cur + timedelta(days=max(1, size) - 1), end)
180
+ out.append((cur.isoformat(), stop.isoformat()))
181
+ cur = stop + timedelta(days=1)
182
+ return out
183
+
184
+ #: ⚠ A REAL CEILING, AND R6's SECOND SENTENCE APPLIES TO IT. Graph pages at 25-100 rows; this is
185
+ #: the number of PAGES a single edge may walk before the loader stops and SAYS it stopped. It is
186
+ #: not a row cap on a connected source (R6 forbids that) — it is a runaway guard, and reaching it
187
+ #: is reported as a problem, never absorbed.
188
+ MAX_PAGES = int(os.environ.get("META_MAX_PAGES") or 200)
189
+ PAGE = int(os.environ.get("META_PAGE_SIZE") or 100)
190
+
191
+
192
+ class MetaError(RuntimeError):
193
+ """A Graph refusal carrying Meta's own words. Safe to print: no token ever reaches it."""
194
+
195
+
196
+ def token():
197
+ """The Meta token from the environment or gitignored `platform/.env`; "" when absent.
198
+
199
+ ⚠ Same resolver `aios-web/api/connectors_meta.py` uses. Duplicated deliberately and minimally:
200
+ `platform/` must not import from `aios-web/api/`, which is the layering rule this repo keeps
201
+ (`core` never imports up). Fifteen lines is the price of that boundary.
202
+ """
203
+ tok = os.environ.get("META_ADS_ACCESS_TOKEN") or ""
204
+ if tok:
205
+ return tok.strip()
206
+ env = _HERE.parent / ".env"
207
+ if env.exists():
208
+ for line in env.read_text(encoding="utf-8", errors="replace").splitlines():
209
+ if line.strip().startswith("META_ADS_ACCESS_TOKEN"):
210
+ _, _, v = line.partition("=")
211
+ return v.strip().strip('"').strip("'")
212
+ return ""
213
+
214
+
215
+ def _get(path, tok, **params):
216
+ params["access_token"] = tok
217
+ url = f"{GRAPH}/{path.lstrip('/')}?" + urllib.parse.urlencode(params)
218
+ req = urllib.request.Request(url, headers={"User-Agent": "aios-meta-store/1"})
219
+ try:
220
+ with urllib.request.urlopen(req, timeout=120) as r:
221
+ return json.loads(r.read().decode("utf-8", "replace"))
222
+ except urllib.error.HTTPError as e:
223
+ try:
224
+ msg = (json.loads(e.read().decode("utf-8", "replace")).get("error") or {}
225
+ ).get("message") or ""
226
+ except Exception:
227
+ msg = ""
228
+ raise MetaError(f"HTTP {e.code} on /{path.lstrip('/')}: {msg[:200]}") from None
229
+ except Exception as e:
230
+ raise MetaError(f"{type(e).__name__} on /{path.lstrip('/')}") from None
231
+
232
+
233
+ #: Graph's own words when a page is too heavy. It arrives as an HTTP **500**, not a 4xx, which is
234
+ #: why it cannot be treated as "the server is broken, give up".
235
+ _TOO_MUCH = "reduce the amount of data"
236
+
237
+ #: ⛔ THE ADS-MANAGEMENT RATE LIMIT, WHICH IS PER AD ACCOUNT AND NOT PER TOKEN. Measured: after a
238
+ #: heavy backfill Graph answers **HTTP 400 "There have been too many calls to this ad-account.
239
+ #: Wait a bit and try again."** It is a 4xx, so nothing about the status code says "retry" — the
240
+ #: MESSAGE is the only signal, which is why it is matched here rather than inferred from a code.
241
+ #: ⚠ It persists for minutes, so the backoff is measured in minutes and BOUNDED: after
242
+ #: `_RATE_TRIES` waits the loader STOPS and reports what it got, rather than sitting in a retry
243
+ #: loop nobody can see. A partial mirror that says it is partial beats a hung sync.
244
+ _RATE_LIMITED = "too many calls"
245
+ _RATE_TRIES = int(os.environ.get("META_RATE_TRIES") or 3)
246
+ _RATE_WAIT = int(os.environ.get("META_RATE_WAIT_S") or 90)
247
+
248
+
249
+ def _walk(path, tok, log, **params):
250
+ """Every page of an edge, paged by CURSOR under our own parameters.
251
+
252
+ ⛔ IT DOES NOT FOLLOW GRAPH'S `paging.next` URL, AND THAT IS THE WHOLE POINT. Measured: the
253
+ first call to `/adcreatives` at limit=100 answers **HTTP 500 "Please reduce the amount of data
254
+ you're asking for"**, the retry at limit=25 succeeds — and then `next` carries the ORIGINAL
255
+ limit=100 and fails again on page 2. A backoff that cannot reach every page is not a backoff.
256
+ Re-issuing each page ourselves with `after=<cursor>` keeps the reduced limit for the whole walk.
257
+
258
+ ⭐ It shrinks the PAGE, never the FIELD LIST. Dropping columns to make a request fit is the
259
+ silent omission R2 forbids, and nothing downstream could see it. Fewer rows per call, always
260
+ every column per row.
261
+
262
+ -> (rows, hit_page_cap)
263
+ """
264
+ out, pages = [], 0
265
+ limit = int(params.pop("limit", None) or PAGE)
266
+ after, waited = None, 0
267
+ while True:
268
+ call = dict(params, limit=limit)
269
+ if after:
270
+ call["after"] = after
271
+ try:
272
+ body = _get(path, tok, **call)
273
+ except MetaError as e:
274
+ if _TOO_MUCH in str(e) and limit > 5:
275
+ limit = max(5, limit // 4)
276
+ log(f" page too heavy for /{path} - retrying at limit={limit} "
277
+ f"(a payload ceiling, not a row cap; every column is still asked for)")
278
+ continue
279
+ if _RATE_LIMITED in str(e).lower() and waited < _RATE_TRIES:
280
+ waited += 1
281
+ log(f" rate-limited on /{path} (per AD ACCOUNT, not per token) - waiting "
282
+ f"{_RATE_WAIT}s, attempt {waited}/{_RATE_TRIES}")
283
+ time.sleep(_RATE_WAIT)
284
+ continue
285
+ if _RATE_LIMITED in str(e).lower():
286
+ log(f" !! GIVING UP on /{path} after {waited} waits: still rate-limited. "
287
+ f"{len(out)} row(s) collected so far are kept. Cause: the ads-management "
288
+ f"limit is per ad account and persists for minutes. Fix: re-run "
289
+ f"`--sync --only <table>` later; the upsert is idempotent.")
290
+ return out, True
291
+ raise
292
+ out.extend(body.get("data") or [])
293
+ pages += 1
294
+ after = ((body.get("paging") or {}).get("cursors") or {}).get("after")
295
+ has_next = bool((body.get("paging") or {}).get("next")) and bool(after)
296
+ if not has_next:
297
+ return out, False
298
+ if pages >= MAX_PAGES:
299
+ log(f" !! STOPPED at MAX_PAGES={MAX_PAGES} on /{path} with more pages left. "
300
+ f"Cause: a runaway guard, not a row cap. Fix: raise META_MAX_PAGES, or narrow the "
301
+ f"window with META_INSIGHTS_DAYS.")
302
+ return out, True
303
+
304
+
305
+ def _cell(value):
306
+ """One Graph value -> one DuckDB cell. Nested structures become compact JSON TEXT rather than
307
+ being dropped: R2 says every field the API returns, and `targeting` is a field."""
308
+ if value is None or isinstance(value, (str, int, float, bool)):
309
+ return value
310
+ return json.dumps(value, separators=(",", ":"), ensure_ascii=False)
311
+
312
+
313
+ def _coltype(name, rows=None):
314
+ """The DuckDB type for one column — decided by the DATA when there is data, by the name only
315
+ as a fallback.
316
+
317
+ ⛔ THE NAME LIST WAS WRONG AND ONLY THE API COULD SAY SO. `cost_per_unique_click`,
318
+ `cost_per_action_type`, `purchase_roas` and friends READ like money and are **arrays of
319
+ action-type objects** on the Insights edge:
320
+ [{"action_type":"outbound_click","value":"1.459854"}]
321
+ Typed DOUBLE from `_DBL`, the insert died with `Conversion Error: Could not convert string
322
+ '[{...}]' to DOUBLE` — after five entity tables had already been written, so the sync looked
323
+ like it worked and then blew up on the last table.
324
+ ⭐ Same principle the field CATALOG is built on, applied one layer down: **ask the response,
325
+ do not assert from a list.** A value that ever arrives as a list or dict is JSON TEXT, because
326
+ that is what `_cell` stores; anything else falls back to the measured name hints.
327
+ """
328
+ if rows:
329
+ seen, kinds = 0, set()
330
+ for r in rows:
331
+ v = r.get(name)
332
+ if v is None or v == "":
333
+ continue
334
+ kinds.add("json" if isinstance(v, (list, dict)) else
335
+ "bool" if isinstance(v, bool) else
336
+ "int" if isinstance(v, int) else
337
+ "float" if isinstance(v, float) else "str")
338
+ seen += 1
339
+ if seen >= 200:
340
+ break
341
+ if kinds:
342
+ if "json" in kinds or "str" in kinds:
343
+ return "VARCHAR" # a JSON blob, or a numeric STRING
344
+ if kinds <= {"int", "bool"}:
345
+ return "BIGINT" if name in _INT else ("VARCHAR" if "bool" in kinds else "BIGINT")
346
+ return "DOUBLE"
347
+ if name in _INT:
348
+ return "BIGINT"
349
+ if name in _DBL:
350
+ return "DOUBLE"
351
+ return "VARCHAR" # ids included — see the header
352
+
353
+
354
+ def _ensure(con, table, fields, rows=None):
355
+ """Create or widen the table, with every column typed from `rows` when they are available.
356
+
357
+ ⚠ AN EXISTING COLUMN WHOSE TYPE IS NOW WRONG IS REBUILT, NOT PATCHED. DuckDB cannot retype a
358
+ column in place, and this table is DERIVED data that can be re-pulled in minutes — so a type
359
+ disagreement drops and recreates rather than limping on with a column that refuses every
360
+ insert. The alternative is a mirror that is permanently unwritable for one bad guess.
361
+ """
362
+ want = {f: _coltype(f, rows) for f in fields}
363
+ have = {r[1]: str(r[2]).upper() for r in con.execute(f"PRAGMA table_info('{table}')").fetchall()}
364
+ if have:
365
+ clash = [f for f, ty in want.items() if f in have and have[f] != ty
366
+ and not (have[f].startswith("VARCHAR") and ty == "VARCHAR")]
367
+ if clash:
368
+ con.execute(f"DROP TABLE {table}")
369
+ have = {}
370
+ if not have:
371
+ cols = ", ".join(f"{f} {want[f]}" for f in fields)
372
+ con.execute(f"CREATE TABLE IF NOT EXISTS {table} (id VARCHAR PRIMARY KEY, {cols})"
373
+ if "id" not in fields else
374
+ f"CREATE TABLE IF NOT EXISTS {table} ({cols})")
375
+ return
376
+ for f in fields:
377
+ if f not in have:
378
+ con.execute(f"ALTER TABLE {table} ADD COLUMN {f} {want[f]}")
379
+
380
+
381
+ def _upsert(con, table, fields, rows):
382
+ """Delete-then-insert by id — the same idempotence `datastore._upsert` gives the Odoo half, so
383
+ a re-sync updates in place and can never append a second copy of the same object."""
384
+ if not rows:
385
+ return 0
386
+ cols = list(fields)
387
+ ids = [str(r.get("id") or "") for r in rows]
388
+ q = ",".join("?" for _ in ids)
389
+ con.execute(f"DELETE FROM {table} WHERE id IN ({q})", ids)
390
+ con.executemany(
391
+ f"INSERT INTO {table} ({', '.join(cols)}) VALUES ({', '.join('?' for _ in cols)})",
392
+ [[_cell(r.get(c)) for c in cols] for r in rows])
393
+ return len(rows)
394
+
395
+
396
+ def sync(tenant_key="royal-imports", tok=None, log=print, insights=True, insights_days=None):
397
+ """Pull every level into the tenant's mirror. -> a report dict; raises only on a bad token.
398
+
399
+ Idempotent: re-running updates rows in place. Safe to call at boot and on the resync loop,
400
+ exactly as `odoo_relational.refresh` is.
401
+ """
402
+ tok = tok or token()
403
+ # ⛔ THE WINDOW IS A PARAMETER, NOT AN ENVIRONMENT READ AT CALL TIME — and the difference cost a
404
+ # live deploy. `INSIGHTS_DAYS` binds at IMPORT (module scope), so `main._pull_meta`'s
405
+ # `os.environ.setdefault("META_INSIGHTS_DAYS", "7")` executed AFTER this module was already
406
+ # imported and changed nothing: every boot pulled **90 days**, not 7, which is the slow path
407
+ # that trips the per-ad-account rate limit and never finishes. The comment beside that call
408
+ # claimed a short window the code could not deliver.
409
+ # ⭐ The shape: **a knob read at import cannot be turned by a caller at runtime.** Passing it
410
+ # makes the caller's intent effective instead of aspirational; the env var stays the DEFAULT.
411
+ days = int(insights_days or INSIGHTS_DAYS)
412
+ report = {"tenant": tenant_key, "tables": {}, "problems": [], "accounts": []}
413
+ if not tok:
414
+ report["problems"].append(
415
+ "META_ADS_ACCESS_TOKEN is not set in the environment or in platform/.env, so nothing "
416
+ "was pulled. This is a missing CREDENTIAL, not a missing capability.")
417
+ return report
418
+
419
+ path = datastore.path_for(tenant_key)
420
+ if Path(datastore.DB_PATH) != Path(path):
421
+ datastore.use_path(path)
422
+ con = datastore.connect()
423
+ log(f" mirror: {Path(path).name}")
424
+
425
+ accts, _ = _walk("me/adaccounts", tok, log, fields="id,name", limit=PAGE)
426
+ report["accounts"] = [a.get("id") for a in accts]
427
+ if not accts:
428
+ report["problems"].append("the token reaches no ad accounts")
429
+ return report
430
+
431
+ # ── the five entity levels ────────────────────────────────────────────────────────────────
432
+ for table, (edge, fields, _parent) in SPECS.items():
433
+ rows, capped = [], False
434
+ for acct in accts:
435
+ aid = acct["id"]
436
+ if edge is None:
437
+ rows.append(_get(aid, tok, fields=",".join(fields)))
438
+ else:
439
+ got, hit = _walk(f"{aid}/{edge}", tok, log, fields=",".join(fields), limit=PAGE)
440
+ rows.extend(got)
441
+ capped = capped or hit
442
+ _ensure(con, table, fields, rows)
443
+ n = _upsert(con, table, fields, rows)
444
+ total = con.execute(f"SELECT count(*) FROM {table}").fetchone()[0]
445
+ report["tables"][table] = {"pulled": n, "in_mirror": total, "capped": capped}
446
+ log(f" {table:<20} pulled {n:>6} mirror total {total:>6}"
447
+ + (" !! PAGE CAP HIT" if capped else ""))
448
+ if capped:
449
+ report["problems"].append(f"{table}: stopped at MAX_PAGES={MAX_PAGES}")
450
+
451
+ # ── the daily Insights grain ──────────────────────────────────────────────────────────────
452
+ if insights:
453
+ rows, capped = [], False
454
+ for acct in accts:
455
+ for since, until in _slices(days, INSIGHTS_SLICE):
456
+ got, hit = _walk(f"{acct['id']}/insights", tok, log,
457
+ fields=",".join(INSIGHT_FIELDS), level=INSIGHTS_LEVEL,
458
+ time_increment="1", limit=INSIGHTS_PAGE,
459
+ time_range=json.dumps({"since": since, "until": until}))
460
+ rows.extend(got)
461
+ capped = capped or hit
462
+ # ⛔ Insights rows carry no id. The key must be STABLE across runs or every re-sync
463
+ # appends a second copy of the same day — so it is composed from the grain itself.
464
+ for r in rows:
465
+ r["id"] = ":".join(str(r.get(k) or "") for k in
466
+ (f"{INSIGHTS_LEVEL}_id", "date_start", "date_stop"))
467
+ fields = ["id"] + INSIGHT_FIELDS
468
+ _ensure(con, INSIGHTS_TABLE, fields, rows)
469
+ n = _upsert(con, INSIGHTS_TABLE, fields, rows)
470
+ total = con.execute(f"SELECT count(*) FROM {INSIGHTS_TABLE}").fetchone()[0]
471
+ report["tables"][INSIGHTS_TABLE] = {"pulled": n, "in_mirror": total, "capped": capped}
472
+ log(f" {INSIGHTS_TABLE:<20} pulled {n:>6} mirror total {total:>6}"
473
+ + (" !! PAGE CAP HIT" if capped else ""))
474
+
475
+ con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
476
+ ["meta", "done", 0, f"{days}d/{INSIGHTS_SLICE}d@{INSIGHTS_LEVEL}",
477
+ sum(t["in_mirror"] for t in report["tables"].values()),
478
+ time.strftime("%Y-%m-%d %H:%M:%S")])
479
+ return report
480
+
481
+
482
+ def status(tenant_key="royal-imports"):
483
+ """What is in the mirror right now, per table. Never fetches."""
484
+ path = datastore.path_for(tenant_key)
485
+ if Path(datastore.DB_PATH) != Path(path):
486
+ datastore.use_path(path)
487
+ out = {}
488
+ con = datastore.connect()
489
+ for table in list(SPECS) + [INSIGHTS_TABLE]:
490
+ try:
491
+ out[table] = con.execute(f"SELECT count(*) FROM {table}").fetchone()[0]
492
+ except Exception:
493
+ out[table] = None # table absent = never synced
494
+ return out
495
+
496
+
497
+ def main(argv=None):
498
+ ap = argparse.ArgumentParser(description="Pull Meta Ads into the tenant's DuckDB mirror.")
499
+ ap.add_argument("--sync", action="store_true")
500
+ ap.add_argument("--status", action="store_true")
501
+ ap.add_argument("--tenant", default="royal-imports")
502
+ ap.add_argument("--no-insights", action="store_true")
503
+ ap.add_argument("--insights-days", type=int, default=None,
504
+ help="override the Insights window for THIS run (default INSIGHTS_DAYS); the env var is the default, this is the caller's say")
505
+ a = ap.parse_args(argv)
506
+ if a.status:
507
+ for k, v in status(a.tenant).items():
508
+ print(f" {k:<20} {'(never synced)' if v is None else v}")
509
+ return 0
510
+ if not a.sync:
511
+ ap.print_help()
512
+ return 2
513
+ rep = sync(a.tenant, insights=not a.no_insights, insights_days=a.insights_days)
514
+ for p in rep["problems"]:
515
+ print(" PROBLEM:", p)
516
+ print(f" accounts: {len(rep['accounts'])} tables: {len(rep['tables'])}")
517
+ return 1 if rep["problems"] else 0
518
+
519
+
520
+ if __name__ == "__main__":
521
+ sys.exit(main())
platform/harness/semantic.py CHANGED
The diff for this file is too large to render. See raw diff
 
platform/model/metrics/sales.yml CHANGED
@@ -1,149 +1,149 @@
1
- # Metrics: sales — the core wholesale metrics, defined ONCE (OM-0). Every surface (pages, the
2
- # metric dictionary, the Analyst, MCP) resolves these by key through harness/semantic.py.
3
- # Fields per metric:
4
- # key/label/description — identity + the human definition (visibility = trust)
5
- # agg + field — sum | count_distinct over the topic's entity
6
- # agg: ratio — numerator/denominator are metric KEYS (resolved recursively)
7
- # agg: derived + expr — arithmetic over metric keys (safe parser; +,-,*,/ and parens only)
8
- # format — usd | int | pct (rendering hint for surfaces)
9
- # ai_context — what a small model must know to use the metric correctly
10
- # validate — the INDEPENDENT Odoo cross-check contract (named method implemented in
11
- # harness/semantic.py _VALIDATORS; a metric that can't tie out says so)
12
- # empty — W37 C1's EMPTY-WINDOW FAMILY. See below; a metric an ENTITY topic
13
- # offers as a lookback column MUST declare one.
14
- #
15
- # ⛔⛔ `empty:` — WHAT A ROW WITH NO ACTIVITY IN THE WINDOW RENDERS AS (wave 37, contract C1).
16
- # Measured and load-bearing: only 1,646 of 5,836 active products sold in the last 90 days, so a
17
- # per-SKU lookback metric has NO GROUP for 72% of the catalogue. Get the default wrong and every
18
- # product grid reads as broken. Two values, and the difference is whether the blank cell would be
19
- # a TRUE STATEMENT:
20
- # empty: zero ADDITIVE — units, revenue, margin $, COGS. "It sold nothing" is a real
21
- # measurement, so 0 is the honest cell and a blank would hide a fact.
22
- # empty: blank RATIO / DERIVED-FROM-A-RATIO — GM %, ASP. A 0% margin on zero sales is a
23
- # FALSE statement, not a missing one. ⛔ AND THE GUARD IS THE DENOMINATOR, NOT
24
- # THE MISSING GROUP: `semantic._post_compute` returns 0.0 for `num/0`, so a SKU
25
- # that DID sell at $0 would print "0.0%" with a group behind it. The resolver
26
- # blanks on a zero denominator, which is the only reading that is never a lie.
27
- # ⚠ A metric with no `empty:` is REFUSED by `entity_measures()` rather than defaulted — a
28
- # defaulted family is a guess about truth, and C1 says a metric that cannot say which family it
29
- # is in does not ship.
30
- topic: sales_lines
31
- metrics:
32
- - key: revenue
33
- label: Revenue
34
- agg: sum
35
- field: price_subtotal
36
- format: usd
37
- empty: zero
38
- description: "Untaxed revenue of confirmed wholesale order lines (the owner's 'sales' number)."
39
- ai_context: "Always untaxed; excludes Amazon (GIFTWARE DEALS) and unconfirmed orders. Filter one BU via team_id (Fisch=5, Royal=6)."
40
- validate:
41
- method: order_level_revenue
42
- note: "Σ line price_subtotal must equal Σ parent-order amount_untaxed under the same scope, to the cent (line vs order basis — the built-in cross-check)."
43
-
44
- - key: units
45
- label: Units sold
46
- agg: sum
47
- field: product_uom_qty
48
- format: int
49
- empty: zero
50
- description: "Total quantity across confirmed wholesale order lines."
51
- ai_context: "Mixed UoMs are summed as ordered quantity; for weight/case analysis convert per product UoM first."
52
-
53
- - key: margin
54
- label: Gross margin $
55
- agg: sum
56
- field: margin
57
- format: usd
58
- empty: zero
59
- description: "Line revenue minus line cost (Odoo Margin module), summed."
60
- ai_context: "margin is read_group-aggregatable (Margin module installed). COGS = revenue - margin. purchase_price is per-UNIT cost — never sum it as a total."
61
-
62
- - key: cogs
63
- label: COGS
64
- agg: derived
65
- expr: "revenue - margin"
66
- format: usd
67
- empty: zero
68
- description: "Cost of goods sold, derived: revenue minus gross margin."
69
- ai_context: "Derived, not pulled — Odoo carries cost on lines as margin; COGS is the difference."
70
-
71
- - key: margin_pct
72
- label: Gross margin %
73
- agg: ratio
74
- numerator: margin
75
- denominator: revenue
76
- format: pct
77
- empty: blank
78
- description: "Gross margin as a share of revenue."
79
- ai_context: "Compare across BUs/categories at the same scope only. ⚠ TWO ZERO-REVENUE BEHAVIOURS, deliberately: the scalar/analyst path returns 0 (semantic._post_compute's guard), while an ENTITY LOOKBACK COLUMN renders BLANK (empty: blank) — on a grid, a printed 0.0% beside 5,836 products would assert a margin nobody measured."
80
-
81
- # ⭐ W37-T10 — ASP, the fifth SHIP-FIRST per-SKU metric (proto/P3). Blended $18.76 over 1,646
82
- # SKUs in the 90 days to 2026-08-19. A RATIO of two same-topic base measures, so it survives a
83
- # GROUPED store_query (only a CROSS-TOPIC component is refused — that is what stops `aov`,
84
- # whose denominator `orders` lives on sales_orders).
85
- # ⚠ UNITS ARE AS-ORDERED, mixed UoM. `units` says so and this inherits it: a SKU sold in cases
86
- # and in singles has an ASP blended across both, which is the true average selling price of a
87
- # line unit and NOT a per-piece price.
88
- - key: asp
89
- label: Average selling price
90
- agg: ratio
91
- numerator: revenue
92
- denominator: units
93
- format: usd
94
- empty: blank
95
- description: "Revenue per unit sold — the blended average selling price."
96
- ai_context: "revenue ÷ units at identical scope. Mixed units of measure are summed as ordered quantity, so this is per ordered unit, not per piece. Blank when nothing sold: an ASP of $0 on zero units is a false statement, not a missing one."
97
-
98
- - key: orders
99
- label: Orders
100
- topic: sales_orders
101
- agg: count
102
- format: int
103
- description: "Confirmed wholesale orders in the window (order-header count — what the scorecards show)."
104
- ai_context: "Order-level count; slightly higher than distinct-orders-from-lines because a few confirmed orders carry zero lines (a data-health artifact the reconciliation contract counts exactly)."
105
- validate:
106
- method: order_count
107
- note: "Order-level count must equal distinct line-parent orders + line-less orders, exactly (surfaces empty orders instead of hiding them in a tolerance)."
108
-
109
- - key: customers
110
- label: Active customers
111
- agg: count_distinct
112
- field: order_partner_id
113
- format: int
114
- empty: zero
115
- description: "Distinct customers with at least one confirmed wholesale order line in the window."
116
- ai_context: "Customer = the order's partner. Agent attribution uses res.partner.agent_ids, NOT the order user_id."
117
-
118
- - key: aov
119
- label: Average order value
120
- agg: ratio
121
- numerator: revenue
122
- denominator: orders
123
- format: usd
124
- description: "Revenue per distinct order."
125
- ai_context: "Ratio of two registered metrics at identical scope; never average per-order averages."
126
-
127
- # ⭐ Wave 21 R2 — the FULLY-INVOICED basis, as separate metrics (owner ruling: picker entries,
128
- # not a basis dropdown). Same topics, same scope, ONE predicate narrower: the order's Odoo
129
- # invoice_status = 'invoiced'. `store_filter_sql` filters the store path (sum/count CASE);
130
- # `live_domain` filters the live path — BOTH or store_parity compares two different questions.
131
- - key: revenue_invoiced
132
- label: Sales — fully invoiced
133
- agg: sum
134
- field: price_subtotal
135
- store_filter_sql: "o.invoice_status = 'invoiced'"
136
- live_domain: [["order_id.invoice_status", "=", "invoiced"]]
137
- format: usd
138
- description: "Untaxed revenue of confirmed wholesale order lines whose parent order Odoo marks fully invoiced (invoice_status = invoiced)."
139
- ai_context: "Same scope as revenue, narrowed by the ORDER-level fully-invoiced flag. This is NOT posted-invoice-line revenue: a partially invoiced order is excluded entirely until Odoo flips the flag. Reconciled store-vs-live per BU by store_parity."
140
-
141
- - key: orders_invoiced
142
- label: Orders — fully invoiced
143
- topic: sales_orders
144
- agg: count
145
- store_filter_sql: "o.invoice_status = 'invoiced'"
146
- live_domain: [["invoice_status", "=", "invoiced"]]
147
- format: int
148
- description: "Confirmed wholesale orders Odoo marks fully invoiced (invoice_status = invoiced)."
149
- ai_context: "Order-header count under the orders scope plus the fully-invoiced flag; partially invoiced and not-yet-invoiced orders are excluded. Reconciled store-vs-live per BU by store_parity."
 
1
+ # Metrics: sales — the core wholesale metrics, defined ONCE (OM-0). Every surface (pages, the
2
+ # metric dictionary, the Analyst, MCP) resolves these by key through harness/semantic.py.
3
+ # Fields per metric:
4
+ # key/label/description — identity + the human definition (visibility = trust)
5
+ # agg + field — sum | count_distinct over the topic's entity
6
+ # agg: ratio — numerator/denominator are metric KEYS (resolved recursively)
7
+ # agg: derived + expr — arithmetic over metric keys (safe parser; +,-,*,/ and parens only)
8
+ # format — usd | int | pct (rendering hint for surfaces)
9
+ # ai_context — what a small model must know to use the metric correctly
10
+ # validate — the INDEPENDENT Odoo cross-check contract (named method implemented in
11
+ # harness/semantic.py _VALIDATORS; a metric that can't tie out says so)
12
+ # empty — W37 C1's EMPTY-WINDOW FAMILY. See below; a metric an ENTITY topic
13
+ # offers as a lookback column MUST declare one.
14
+ #
15
+ # ⛔⛔ `empty:` — WHAT A ROW WITH NO ACTIVITY IN THE WINDOW RENDERS AS (wave 37, contract C1).
16
+ # Measured and load-bearing: only 1,646 of 5,836 active products sold in the last 90 days, so a
17
+ # per-SKU lookback metric has NO GROUP for 72% of the catalogue. Get the default wrong and every
18
+ # product grid reads as broken. Two values, and the difference is whether the blank cell would be
19
+ # a TRUE STATEMENT:
20
+ # empty: zero ADDITIVE — units, revenue, margin $, COGS. "It sold nothing" is a real
21
+ # measurement, so 0 is the honest cell and a blank would hide a fact.
22
+ # empty: blank RATIO / DERIVED-FROM-A-RATIO — GM %, ASP. A 0% margin on zero sales is a
23
+ # FALSE statement, not a missing one. ⛔ AND THE GUARD IS THE DENOMINATOR, NOT
24
+ # THE MISSING GROUP: `semantic._post_compute` returns 0.0 for `num/0`, so a SKU
25
+ # that DID sell at $0 would print "0.0%" with a group behind it. The resolver
26
+ # blanks on a zero denominator, which is the only reading that is never a lie.
27
+ # ⚠ A metric with no `empty:` is REFUSED by `entity_measures()` rather than defaulted — a
28
+ # defaulted family is a guess about truth, and C1 says a metric that cannot say which family it
29
+ # is in does not ship.
30
+ topic: sales_lines
31
+ metrics:
32
+ - key: revenue
33
+ label: Revenue
34
+ agg: sum
35
+ field: price_subtotal
36
+ format: usd
37
+ empty: zero
38
+ description: "Untaxed revenue of confirmed wholesale order lines (the owner's 'sales' number)."
39
+ ai_context: "Always untaxed; excludes Amazon (GIFTWARE DEALS) and unconfirmed orders. Filter one BU via team_id (Fisch=5, Royal=6)."
40
+ validate:
41
+ method: order_level_revenue
42
+ note: "Σ line price_subtotal must equal Σ parent-order amount_untaxed under the same scope, to the cent (line vs order basis — the built-in cross-check)."
43
+
44
+ - key: units
45
+ label: Units sold
46
+ agg: sum
47
+ field: product_uom_qty
48
+ format: int
49
+ empty: zero
50
+ description: "Total quantity across confirmed wholesale order lines."
51
+ ai_context: "Mixed UoMs are summed as ordered quantity; for weight/case analysis convert per product UoM first."
52
+
53
+ - key: margin
54
+ label: Gross margin $
55
+ agg: sum
56
+ field: margin
57
+ format: usd
58
+ empty: zero
59
+ description: "Line revenue minus line cost (Odoo Margin module), summed."
60
+ ai_context: "margin is read_group-aggregatable (Margin module installed). COGS = revenue - margin. purchase_price is per-UNIT cost — never sum it as a total."
61
+
62
+ - key: cogs
63
+ label: COGS
64
+ agg: derived
65
+ expr: "revenue - margin"
66
+ format: usd
67
+ empty: zero
68
+ description: "Cost of goods sold, derived: revenue minus gross margin."
69
+ ai_context: "Derived, not pulled — Odoo carries cost on lines as margin; COGS is the difference."
70
+
71
+ - key: margin_pct
72
+ label: Gross margin %
73
+ agg: ratio
74
+ numerator: margin
75
+ denominator: revenue
76
+ format: pct
77
+ empty: blank
78
+ description: "Gross margin as a share of revenue."
79
+ ai_context: "Compare across BUs/categories at the same scope only. ⚠ TWO ZERO-REVENUE BEHAVIOURS, deliberately: the scalar/analyst path returns 0 (semantic._post_compute's guard), while an ENTITY LOOKBACK COLUMN renders BLANK (empty: blank) — on a grid, a printed 0.0% beside 5,836 products would assert a margin nobody measured."
80
+
81
+ # ⭐ W37-T10 — ASP, the fifth SHIP-FIRST per-SKU metric (proto/P3). Blended $18.76 over 1,646
82
+ # SKUs in the 90 days to 2026-08-19. A RATIO of two same-topic base measures, so it survives a
83
+ # GROUPED store_query (only a CROSS-TOPIC component is refused — that is what stops `aov`,
84
+ # whose denominator `orders` lives on sales_orders).
85
+ # ⚠ UNITS ARE AS-ORDERED, mixed UoM. `units` says so and this inherits it: a SKU sold in cases
86
+ # and in singles has an ASP blended across both, which is the true average selling price of a
87
+ # line unit and NOT a per-piece price.
88
+ - key: asp
89
+ label: Average selling price
90
+ agg: ratio
91
+ numerator: revenue
92
+ denominator: units
93
+ format: usd
94
+ empty: blank
95
+ description: "Revenue per unit sold — the blended average selling price."
96
+ ai_context: "revenue ÷ units at identical scope. Mixed units of measure are summed as ordered quantity, so this is per ordered unit, not per piece. Blank when nothing sold: an ASP of $0 on zero units is a false statement, not a missing one."
97
+
98
+ - key: orders
99
+ label: Orders
100
+ topic: sales_orders
101
+ agg: count
102
+ format: int
103
+ description: "Confirmed wholesale orders in the window (order-header count — what the scorecards show)."
104
+ ai_context: "Order-level count; slightly higher than distinct-orders-from-lines because a few confirmed orders carry zero lines (a data-health artifact the reconciliation contract counts exactly)."
105
+ validate:
106
+ method: order_count
107
+ note: "Order-level count must equal distinct line-parent orders + line-less orders, exactly (surfaces empty orders instead of hiding them in a tolerance)."
108
+
109
+ - key: customers
110
+ label: Active customers
111
+ agg: count_distinct
112
+ field: order_partner_id
113
+ format: int
114
+ empty: zero
115
+ description: "Distinct customers with at least one confirmed wholesale order line in the window."
116
+ ai_context: "Customer = the order's partner. Agent attribution uses res.partner.agent_ids, NOT the order user_id."
117
+
118
+ - key: aov
119
+ label: Average order value
120
+ agg: ratio
121
+ numerator: revenue
122
+ denominator: orders
123
+ format: usd
124
+ description: "Revenue per distinct order."
125
+ ai_context: "Ratio of two registered metrics at identical scope; never average per-order averages."
126
+
127
+ # ⭐ Wave 21 R2 — the FULLY-INVOICED basis, as separate metrics (owner ruling: picker entries,
128
+ # not a basis dropdown). Same topics, same scope, ONE predicate narrower: the order's Odoo
129
+ # invoice_status = 'invoiced'. `store_filter_sql` filters the store path (sum/count CASE);
130
+ # `live_domain` filters the live path — BOTH or store_parity compares two different questions.
131
+ - key: revenue_invoiced
132
+ label: Sales — fully invoiced
133
+ agg: sum
134
+ field: price_subtotal
135
+ store_filter_sql: "o.invoice_status = 'invoiced'"
136
+ live_domain: [["order_id.invoice_status", "=", "invoiced"]]
137
+ format: usd
138
+ description: "Untaxed revenue of confirmed wholesale order lines whose parent order Odoo marks fully invoiced (invoice_status = invoiced)."
139
+ ai_context: "Same scope as revenue, narrowed by the ORDER-level fully-invoiced flag. This is NOT posted-invoice-line revenue: a partially invoiced order is excluded entirely until Odoo flips the flag. Reconciled store-vs-live per BU by store_parity."
140
+
141
+ - key: orders_invoiced
142
+ label: Orders — fully invoiced
143
+ topic: sales_orders
144
+ agg: count
145
+ store_filter_sql: "o.invoice_status = 'invoiced'"
146
+ live_domain: [["invoice_status", "=", "invoiced"]]
147
+ format: int
148
+ description: "Confirmed wholesale orders Odoo marks fully invoiced (invoice_status = invoiced)."
149
+ ai_context: "Order-header count under the orders scope plus the fully-invoiced flag; partially invoiced and not-yet-invoiced orders are excluded. Reconciled store-vs-live per BU by store_parity."
platform/model/topics/odoo_accounts.yml CHANGED
@@ -1,60 +1,60 @@
1
- # ⭐⭐ W33-T49 (owner item 13) — an ENTITY topic: the semantic layer can finally see
2
- # the databases a person actually opens, not only the line and document grains.
3
- #
4
- # ⛔ THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
5
- # and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
6
- # field row: add the column to the grid contract and re-emit, or the agent is being
7
- # trained on a schema the product does not have.
8
- key: odoo_accounts
9
- label: "Odoo GL accounts"
10
- entity: account.account
11
- # The database this topic DESCRIBES — the same store key the nav opens (W33-T46).
12
- grid: ut_odoo_accounts
13
- subject: "odoo:account.account"
14
- grain: "one row per GL account in the chart of accounts"
15
- scope:
16
- population: "every account.account record (192 measured)"
17
- store:
18
- table: account_account
19
- alias: a
20
- # NO date_col — a registry is not a dated event stream. Stated rather than
21
- # omitted, so its absence reads as a fact and not as an unfinished file.
22
- dims:
23
- account_type: {label: "Account type"}
24
-
25
- # key / label / type / kind, derived from the grid contract. `kind` says where the
26
- # value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
27
- # from another topic; a `link` points at another database.
28
- fields:
29
- - key: account_code
30
- label: "Code"
31
- type: text
32
- kind: data
33
- - key: account_name
34
- label: "Account"
35
- type: text
36
- kind: data
37
- - key: odoo_id
38
- label: "Odoo ID"
39
- type: int
40
- kind: data
41
- means: "The `account.account` id. Also this row's id."
42
- - key: account_type
43
- label: "Type"
44
- type: select
45
- kind: data
46
- - key: is_expense
47
- label: "Expense account"
48
- type: checkbox
49
- kind: data
50
- means: "Ticked for the expense family - the same predicate the semantic layer's gl_lines topic uses, so this column and that topic cannot disagree."
51
- - key: refreshed
52
- label: "Refreshed"
53
- type: date
54
- kind: data
55
- means: "When this row was last reconciled against Odoo."
56
-
57
- ai_context: >
58
- The chart of accounts — the registry gl_lines posts against.
59
- Use it to resolve an account code to a name or a type.
60
- ⚠ `is_expense` is a DERIVED boolean standing in for Odoo's five-value `internal_group` (asset/liability/equity/income/expense), which the mirror does not yet carry.
 
1
+ # ⭐⭐ W33-T49 (owner item 13) — an ENTITY topic: the semantic layer can finally see
2
+ # the databases a person actually opens, not only the line and document grains.
3
+ #
4
+ # ⛔ THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
5
+ # and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
6
+ # field row: add the column to the grid contract and re-emit, or the agent is being
7
+ # trained on a schema the product does not have.
8
+ key: odoo_accounts
9
+ label: "Odoo GL accounts"
10
+ entity: account.account
11
+ # The database this topic DESCRIBES — the same store key the nav opens (W33-T46).
12
+ grid: ut_odoo_accounts
13
+ subject: "odoo:account.account"
14
+ grain: "one row per GL account in the chart of accounts"
15
+ scope:
16
+ population: "every account.account record (192 measured)"
17
+ store:
18
+ table: account_account
19
+ alias: a
20
+ # NO date_col — a registry is not a dated event stream. Stated rather than
21
+ # omitted, so its absence reads as a fact and not as an unfinished file.
22
+ dims:
23
+ account_type: {label: "Account type"}
24
+
25
+ # key / label / type / kind, derived from the grid contract. `kind` says where the
26
+ # value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
27
+ # from another topic; a `link` points at another database.
28
+ fields:
29
+ - key: account_code
30
+ label: "Code"
31
+ type: text
32
+ kind: data
33
+ - key: account_name
34
+ label: "Account"
35
+ type: text
36
+ kind: data
37
+ - key: odoo_id
38
+ label: "Odoo ID"
39
+ type: int
40
+ kind: data
41
+ means: "The `account.account` id. Also this row's id."
42
+ - key: account_type
43
+ label: "Type"
44
+ type: select
45
+ kind: data
46
+ - key: is_expense
47
+ label: "Expense account"
48
+ type: checkbox
49
+ kind: data
50
+ means: "Ticked for the expense family - the same predicate the semantic layer's gl_lines topic uses, so this column and that topic cannot disagree."
51
+ - key: refreshed
52
+ label: "Refreshed"
53
+ type: date
54
+ kind: data
55
+ means: "When this row was last reconciled against Odoo."
56
+
57
+ ai_context: >
58
+ The chart of accounts — the registry gl_lines posts against.
59
+ Use it to resolve an account code to a name or a type.
60
+ ⚠ `is_expense` is a DERIVED boolean standing in for Odoo's five-value `internal_group` (asset/liability/equity/income/expense), which the mirror does not yet carry.
platform/model/topics/odoo_agents.yml CHANGED
@@ -1,91 +1,91 @@
1
- # ⭐⭐ W33-T49 (owner item 13) — an ENTITY topic: the semantic layer can finally see
2
- # the databases a person actually opens, not only the line and document grains.
3
- #
4
- # ⛔ THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
5
- # and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
6
- # field row: add the column to the grid contract and re-emit, or the agent is being
7
- # trained on a schema the product does not have.
8
- key: odoo_agents
9
- label: "Odoo agents"
10
- entity: res.partner
11
- # The database this topic DESCRIBES — the same store key the nav opens (W33-T46).
12
- grid: ut_odoo_agents
13
- subject: "odoo:res.partner.agent"
14
- grain: "one row per sales agent"
15
- scope:
16
- population: "a UNION of two disagreeing sources — partners carrying commission lines, and partners flagged res_partner.agent = TRUE"
17
- why_union: "measured 2026-08-09: 16 carry commission lines, 17 carry the flag, and the union is 19 — either source alone silently drops real agents"
18
- store:
19
- table: res_partner
20
- alias: p
21
- # NO date_col — a registry is not a dated event stream. Stated rather than
22
- # omitted, so its absence reads as a fact and not as an unfinished file.
23
-
24
- # ⭐⭐ W37-T12 / owner item 4 (R1) — THE LOOKBACK-MEASURE BINDING, the agent half of
25
- # *"It should apply to Odoo agents database as well."* Same contract as
26
- # `odoo_products.yml`: a FACT topic, and the dim of that topic which carries THIS
27
- # entity's identity.
28
- #
29
- # ⛔⛔ WHICH AGENT SOURCE, STATED — because there are THREE in this Odoo and they name
30
- # DIFFERENT PEOPLE (`proto/P3-metric-catalog.md`: 9 agents carry route-1 revenue, 12
31
- # carry commission lines, 17 carry the flag). This binds ROUTE 1, the CUSTOMER-MASTER
32
- # BOOK: `sales_lines.agent` is `rp.agent_id`, the customer's assigned agent, so every
33
- # order of that customer counts toward their agent. That is "whose book is this" and it
34
- # is the right question for a column on the AGENT REGISTRY.
35
- # ⚠ It is NOT commission. `account.invoice.line.agent` (topic `commission_lines`) is the
36
- # per-INVOICE-LINE credited agent and answers a different question; `sales_lines.yml`'s
37
- # own `agent` dim comment carries the full disagreement and says never to "fix" one by
38
- # reading the other.
39
- #
40
- # ⭐ THE JOIN NEEDS NO NEW DIM, unlike the product side. `sales_lines.agent` declares a
41
- # `name_col`, so `store_query` emits `agent_id` — which IS `res.partner.id`, which IS
42
- # this grid's `odoo_id` identity. Product needed `product_code` because its grid keys on
43
- # `default_code`; this one already keys on the same integer the fact topic groups by.
44
- measures:
45
- source: sales_lines
46
- dim: agent
47
- keys: [revenue, units, margin, cogs, margin_pct, asp, customers]
48
- not_yet:
49
- - key: orders / aov
50
- cause: "`orders` lives on topic `sales_orders`, so under a `group_by` it is a CROSS-TOPIC measure and `semantic.store_query` refuses it (scalar-only). `aov` inherits the refusal through its denominator"
51
- fix: "add an ORDER-COUNT metric to `sales_lines` itself — `count_distinct` over `l.order_id` — which answers the same question at this grain in one pass"
52
- - key: commission_amount
53
- cause: "the commission basis lives on `account.invoice.line.agent` (topic `commission_lines`), a different topic AND a different grain; `invoice_lines` deduplicates to at most one agent per line"
54
- fix: "bind a SECOND measures block per source once the contract allows more than one, and label the columns so the two routes can never be read as the same number"
55
-
56
- # key / label / type / kind, derived from the grid contract. `kind` says where the
57
- # value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
58
- # from another topic; a `link` points at another database.
59
- fields:
60
- - key: agent
61
- label: "Agent"
62
- type: text
63
- kind: data
64
- - key: odoo_id
65
- label: "Odoo ID"
66
- type: int
67
- kind: data
68
- means: "The `res.partner` id. Also this row's id."
69
- - key: agent_id
70
- label: "Odoo agent id"
71
- type: int
72
- kind: data
73
- - key: flagged
74
- label: "Flagged in Odoo"
75
- type: checkbox
76
- kind: data
77
- means: "Ticked = `res.partner.agent` is set. Unticked agents were found by their commission lines instead - both are real, which is why this table is the union of the two."
78
- - key: commissioned
79
- label: "Has commission lines"
80
- type: checkbox
81
- kind: data
82
- - key: refreshed
83
- label: "Refreshed"
84
- type: date
85
- kind: data
86
- means: "When this row was last reconciled against Odoo."
87
-
88
- ai_context: >
89
- The sales-agent registry.
90
- `flagged` and `commissioned` are the two SOURCES, kept as separate columns rather than merged, because they disagree and the disagreement is information.
91
- For an agent's BOOK use sales_lines/commission_lines with the agent dim.
 
1
+ # ⭐⭐ W33-T49 (owner item 13) — an ENTITY topic: the semantic layer can finally see
2
+ # the databases a person actually opens, not only the line and document grains.
3
+ #
4
+ # ⛔ THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
5
+ # and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
6
+ # field row: add the column to the grid contract and re-emit, or the agent is being
7
+ # trained on a schema the product does not have.
8
+ key: odoo_agents
9
+ label: "Odoo agents"
10
+ entity: res.partner
11
+ # The database this topic DESCRIBES — the same store key the nav opens (W33-T46).
12
+ grid: ut_odoo_agents
13
+ subject: "odoo:res.partner.agent"
14
+ grain: "one row per sales agent"
15
+ scope:
16
+ population: "a UNION of two disagreeing sources — partners carrying commission lines, and partners flagged res_partner.agent = TRUE"
17
+ why_union: "measured 2026-08-09: 16 carry commission lines, 17 carry the flag, and the union is 19 — either source alone silently drops real agents"
18
+ store:
19
+ table: res_partner
20
+ alias: p
21
+ # NO date_col — a registry is not a dated event stream. Stated rather than
22
+ # omitted, so its absence reads as a fact and not as an unfinished file.
23
+
24
+ # ⭐⭐ W37-T12 / owner item 4 (R1) — THE LOOKBACK-MEASURE BINDING, the agent half of
25
+ # *"It should apply to Odoo agents database as well."* Same contract as
26
+ # `odoo_products.yml`: a FACT topic, and the dim of that topic which carries THIS
27
+ # entity's identity.
28
+ #
29
+ # ⛔⛔ WHICH AGENT SOURCE, STATED — because there are THREE in this Odoo and they name
30
+ # DIFFERENT PEOPLE (`proto/P3-metric-catalog.md`: 9 agents carry route-1 revenue, 12
31
+ # carry commission lines, 17 carry the flag). This binds ROUTE 1, the CUSTOMER-MASTER
32
+ # BOOK: `sales_lines.agent` is `rp.agent_id`, the customer's assigned agent, so every
33
+ # order of that customer counts toward their agent. That is "whose book is this" and it
34
+ # is the right question for a column on the AGENT REGISTRY.
35
+ # ⚠ It is NOT commission. `account.invoice.line.agent` (topic `commission_lines`) is the
36
+ # per-INVOICE-LINE credited agent and answers a different question; `sales_lines.yml`'s
37
+ # own `agent` dim comment carries the full disagreement and says never to "fix" one by
38
+ # reading the other.
39
+ #
40
+ # ⭐ THE JOIN NEEDS NO NEW DIM, unlike the product side. `sales_lines.agent` declares a
41
+ # `name_col`, so `store_query` emits `agent_id` — which IS `res.partner.id`, which IS
42
+ # this grid's `odoo_id` identity. Product needed `product_code` because its grid keys on
43
+ # `default_code`; this one already keys on the same integer the fact topic groups by.
44
+ measures:
45
+ source: sales_lines
46
+ dim: agent
47
+ keys: [revenue, units, margin, cogs, margin_pct, asp, customers]
48
+ not_yet:
49
+ - key: orders / aov
50
+ cause: "`orders` lives on topic `sales_orders`, so under a `group_by` it is a CROSS-TOPIC measure and `semantic.store_query` refuses it (scalar-only). `aov` inherits the refusal through its denominator"
51
+ fix: "add an ORDER-COUNT metric to `sales_lines` itself — `count_distinct` over `l.order_id` — which answers the same question at this grain in one pass"
52
+ - key: commission_amount
53
+ cause: "the commission basis lives on `account.invoice.line.agent` (topic `commission_lines`), a different topic AND a different grain; `invoice_lines` deduplicates to at most one agent per line"
54
+ fix: "bind a SECOND measures block per source once the contract allows more than one, and label the columns so the two routes can never be read as the same number"
55
+
56
+ # key / label / type / kind, derived from the grid contract. `kind` says where the
57
+ # value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
58
+ # from another topic; a `link` points at another database.
59
+ fields:
60
+ - key: agent
61
+ label: "Agent"
62
+ type: text
63
+ kind: data
64
+ - key: odoo_id
65
+ label: "Odoo ID"
66
+ type: int
67
+ kind: data
68
+ means: "The `res.partner` id. Also this row's id."
69
+ - key: agent_id
70
+ label: "Odoo agent id"
71
+ type: int
72
+ kind: data
73
+ - key: flagged
74
+ label: "Flagged in Odoo"
75
+ type: checkbox
76
+ kind: data
77
+ means: "Ticked = `res.partner.agent` is set. Unticked agents were found by their commission lines instead - both are real, which is why this table is the union of the two."
78
+ - key: commissioned
79
+ label: "Has commission lines"
80
+ type: checkbox
81
+ kind: data
82
+ - key: refreshed
83
+ label: "Refreshed"
84
+ type: date
85
+ kind: data
86
+ means: "When this row was last reconciled against Odoo."
87
+
88
+ ai_context: >
89
+ The sales-agent registry.
90
+ `flagged` and `commissioned` are the two SOURCES, kept as separate columns rather than merged, because they disagree and the disagreement is information.
91
+ For an agent's BOOK use sales_lines/commission_lines with the agent dim.
platform/model/topics/odoo_bills.yml CHANGED
@@ -1,86 +1,86 @@
1
- # ⭐⭐ W33-T49 (owner item 13) — an ENTITY topic: the semantic layer can finally see
2
- # the databases a person actually opens, not only the line and document grains.
3
- #
4
- # ⛔ THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
5
- # and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
6
- # field row: add the column to the grid contract and re-emit, or the agent is being
7
- # trained on a schema the product does not have.
8
- key: odoo_bills
9
- label: "Odoo vendor bills"
10
- entity: account.move
11
- # The database this topic DESCRIBES — the same store key the nav opens (W33-T46).
12
- grid: ut_odoo_bills
13
- subject: "odoo:account.move.vendor"
14
- grain: "one row per posted VENDOR bill or vendor credit note (document grain)"
15
- scope:
16
- population: "move_type in (in_invoice, in_refund) AND state = posted"
17
- signs: "for payables the residual is negative on the Odoo side; take the absolute value for an AP figure"
18
- store:
19
- table: account_move
20
- alias: m
21
- date_col: "m.invoice_date"
22
- dims:
23
- vendor: {label: "Vendor"}
24
- payment_state: {label: "Payment state"}
25
- move_type: {label: "Document type"}
26
-
27
- # key / label / type / kind, derived from the grid contract. `kind` says where the
28
- # value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
29
- # from another topic; a `link` points at another database.
30
- fields:
31
- - key: bill_no
32
- label: "Bill"
33
- type: text
34
- kind: data
35
- - key: odoo_id
36
- label: "Odoo ID"
37
- type: int
38
- kind: data
39
- means: "The `account.move` id. Also this row's id."
40
- - key: vendor
41
- label: "Vendor"
42
- type: text
43
- kind: data
44
- - key: vendor_id
45
- label: "Odoo vendor id"
46
- type: int
47
- kind: data
48
- - key: invoice_date
49
- label: "Bill date"
50
- type: date
51
- kind: data
52
- - key: due_date
53
- label: "Due date"
54
- type: date
55
- kind: data
56
- - key: amount_untaxed
57
- label: "Billed $"
58
- type: currency
59
- kind: data
60
- - key: residual
61
- label: "Outstanding $"
62
- type: currency
63
- kind: data
64
- - key: payment_state
65
- label: "Payment state"
66
- type: select
67
- kind: data
68
- - key: move_type
69
- label: "Document"
70
- type: select
71
- kind: data
72
- - key: vendor_link
73
- label: "Vendor record"
74
- type: link
75
- kind: link
76
- to_grid: ut_odoo_vendors
77
- - key: refreshed
78
- label: "Refreshed"
79
- type: date
80
- kind: data
81
- means: "When this row was last reconciled against Odoo."
82
-
83
- ai_context: >
84
- Vendor bills at DOCUMENT grain — the payables side of account_move.
85
- Use it for AP questions.
86
- ⚠ The census found `ref` (the VENDOR's own invoice number), `journal_id` and `currency_id` populated in Odoo and absent from the mirror, so they cannot be answered from here yet.
 
1
+ # ⭐⭐ W33-T49 (owner item 13) — an ENTITY topic: the semantic layer can finally see
2
+ # the databases a person actually opens, not only the line and document grains.
3
+ #
4
+ # ⛔ THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
5
+ # and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
6
+ # field row: add the column to the grid contract and re-emit, or the agent is being
7
+ # trained on a schema the product does not have.
8
+ key: odoo_bills
9
+ label: "Odoo vendor bills"
10
+ entity: account.move
11
+ # The database this topic DESCRIBES — the same store key the nav opens (W33-T46).
12
+ grid: ut_odoo_bills
13
+ subject: "odoo:account.move.vendor"
14
+ grain: "one row per posted VENDOR bill or vendor credit note (document grain)"
15
+ scope:
16
+ population: "move_type in (in_invoice, in_refund) AND state = posted"
17
+ signs: "for payables the residual is negative on the Odoo side; take the absolute value for an AP figure"
18
+ store:
19
+ table: account_move
20
+ alias: m
21
+ date_col: "m.invoice_date"
22
+ dims:
23
+ vendor: {label: "Vendor"}
24
+ payment_state: {label: "Payment state"}
25
+ move_type: {label: "Document type"}
26
+
27
+ # key / label / type / kind, derived from the grid contract. `kind` says where the
28
+ # value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
29
+ # from another topic; a `link` points at another database.
30
+ fields:
31
+ - key: bill_no
32
+ label: "Bill"
33
+ type: text
34
+ kind: data
35
+ - key: odoo_id
36
+ label: "Odoo ID"
37
+ type: int
38
+ kind: data
39
+ means: "The `account.move` id. Also this row's id."
40
+ - key: vendor
41
+ label: "Vendor"
42
+ type: text
43
+ kind: data
44
+ - key: vendor_id
45
+ label: "Odoo vendor id"
46
+ type: int
47
+ kind: data
48
+ - key: invoice_date
49
+ label: "Bill date"
50
+ type: date
51
+ kind: data
52
+ - key: due_date
53
+ label: "Due date"
54
+ type: date
55
+ kind: data
56
+ - key: amount_untaxed
57
+ label: "Billed $"
58
+ type: currency
59
+ kind: data
60
+ - key: residual
61
+ label: "Outstanding $"
62
+ type: currency
63
+ kind: data
64
+ - key: payment_state
65
+ label: "Payment state"
66
+ type: select
67
+ kind: data
68
+ - key: move_type
69
+ label: "Document"
70
+ type: select
71
+ kind: data
72
+ - key: vendor_link
73
+ label: "Vendor record"
74
+ type: link
75
+ kind: link
76
+ to_grid: ut_odoo_vendors
77
+ - key: refreshed
78
+ label: "Refreshed"
79
+ type: date
80
+ kind: data
81
+ means: "When this row was last reconciled against Odoo."
82
+
83
+ ai_context: >
84
+ Vendor bills at DOCUMENT grain — the payables side of account_move.
85
+ Use it for AP questions.
86
+ ⚠ The census found `ref` (the VENDOR's own invoice number), `journal_id` and `currency_id` populated in Odoo and absent from the mirror, so they cannot be answered from here yet.
platform/model/topics/odoo_customers.yml CHANGED
@@ -1,213 +1,213 @@
1
- # ⭐⭐ W33-T49 (owner item 13) — an ENTITY topic: the semantic layer can finally see
2
- # the databases a person actually opens, not only the line and document grains.
3
- #
4
- # ⛔ THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
5
- # and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
6
- # field row: add the column to the grid contract and re-emit, or the agent is being
7
- # trained on a schema the product does not have.
8
- key: odoo_customers
9
- label: "Odoo customers"
10
- entity: res.partner
11
- # The database this topic DESCRIBES — the same store key the nav opens (W33-T46).
12
- grid: customer_data
13
- subject: "odoo:res.partner"
14
- grain: "one row per customer in tenant #0's scoped book — a REGISTRY, not a dated event stream. Identity is the Odoo `res.partner` id (the row's `pid` IS that id)."
15
- scope:
16
- population: "the customer book `modules/customer_data.pool()` builds; the grid's own contract states it as one row per customer who ordered in the last 24 months"
17
- identity: "the Odoo res.partner id — exposed as the `partner_id` column (W33-T43) and used as the row pid, so there is ONE id per row and no second one"
18
- merged: "W33-T44 retired `ut_odoo_customers`, which presented this same subject. This topic describes the SURVIVING database, `customer_data`, which keeps its store bucket so no saved view, grant, cohort or formula moved"
19
- no_date: "a registry has no date dimension; ask sales_lines or customer_invoices for anything time-windowed about a customer"
20
- store:
21
- table: res_partner
22
- alias: p
23
- # NO date_col — a registry is not a dated event stream. Stated rather than
24
- # omitted, so its absence reads as a fact and not as an unfinished file.
25
- dims:
26
- agent: {label: "Sales agent"}
27
- state: {label: "State"}
28
- country: {label: "Country"}
29
-
30
- # key / label / type / kind, derived from the grid contract. `kind` says where the
31
- # value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
32
- # from another topic; a `link` points at another database.
33
- fields:
34
- - key: customer
35
- label: "Customer"
36
- type: text
37
- kind: data
38
- means: "The customer's name in Odoo. One row per customer who ordered in the last 24 months."
39
- - key: partner_id
40
- label: "Odoo ID"
41
- type: int
42
- kind: data
43
- means: "The Odoo res.partner id — this row's identity, and the key every Odoo document joins on. W33-T43 / owner item 12: 'One unique ID per database always.' It is DERIVED rather than read off the pool row because a customer row's pid IS the partner id, so the value is already on every row and a second copy in the pool would be a second source for one fact."
44
- - key: odoo_status
45
- label: "Odoo record"
46
- type: status
47
- kind: data
48
- means: "Whether this customer still exists in Odoo. Archived means deleted there."
49
- - key: agent
50
- label: "Agent"
51
- type: text
52
- kind: data
53
- means: "The sales agent who owns this account."
54
- - key: dba
55
- label: "DBA"
56
- type: select
57
- kind: data
58
- means: "The brand this customer buys from - Fisch, Royal, or both. Amazon-channel orders are not a DBA."
59
- - key: salesperson
60
- label: "Salesperson"
61
- type: text
62
- kind: data
63
- means: "Who keyed in most of this customer's orders — not the Agent, who owns the account."
64
- - key: street
65
- label: "Street"
66
- type: text
67
- kind: data
68
- means: "First address line, from res.partner directly - not the geocoder, so a customer the map cannot place still shows its address."
69
- - key: street2
70
- label: "Street 2"
71
- type: text
72
- kind: data
73
- means: "Second address line (suite, unit, floor) on the customer's Odoo address."
74
- - key: city
75
- label: "City"
76
- type: text
77
- kind: data
78
- means: "City on the customer's Odoo address."
79
- - key: state
80
- label: "State"
81
- type: text
82
- kind: data
83
- means: "State or province on the customer's Odoo address."
84
- - key: country
85
- label: "Country"
86
- type: text
87
- kind: data
88
- means: "Country on the customer's Odoo address."
89
- - key: zip
90
- label: "ZIP"
91
- type: text
92
- kind: data
93
- means: "Postal code on the customer's Odoo address."
94
- - key: customer_since
95
- label: "Customer since"
96
- type: date
97
- kind: data
98
- means: "When this customer was first set up in Odoo."
99
- - key: tags
100
- label: "Tags"
101
- type: text
102
- kind: data
103
- means: "Odoo labels on this customer, comma-separated."
104
- - key: pricelist
105
- label: "Price list"
106
- type: text
107
- kind: data
108
- means: "The price list this customer buys on."
109
- - key: payment_terms
110
- label: "Payment terms"
111
- type: text
112
- kind: data
113
- means: "Payment terms on this customer's account — Net 30, for example."
114
- - key: last_order
115
- label: "Last order"
116
- type: date
117
- kind: data
118
- means: "Date of the most recent confirmed order."
119
- - key: overdue_days
120
- label: "Overdue days"
121
- type: int
122
- kind: data
123
- means: "How many days late this customer is running against their own usual ordering rhythm."
124
- - key: est_missed
125
- label: "Est. missed $"
126
- type: currency
127
- kind: data
128
- means: "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed."
129
- - key: ar_open
130
- label: "AR current $"
131
- type: currency
132
- kind: data
133
- means: "Invoiced money owed but not yet due (a 5-day grace applies before it counts as overdue)."
134
- - key: ar_overdue
135
- label: "AR overdue $"
136
- type: currency
137
- kind: data
138
- means: "Invoiced money past due — same basis as the Collections page."
139
- - key: ar_outstanding
140
- label: "AR outstanding $"
141
- type: currency
142
- kind: data
143
- means: "Total invoiced money owed right now: AR current $ plus AR overdue $."
144
- - key: ar_exposure
145
- label: "Credit exposure $"
146
- type: currency
147
- kind: data
148
- means: "The most you could be out if they stopped paying today: open, overdue, draft and not-yet-invoiced."
149
- - key: ar_aged_1_30
150
- label: "1-30 days $"
151
- type: currency
152
- kind: data
153
- means: "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $."
154
- - key: ar_aged_31_60
155
- label: "31-60 days $"
156
- type: currency
157
- kind: data
158
- means: "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $."
159
- - key: ar_aged_61_90
160
- label: "61-90 days $"
161
- type: currency
162
- kind: data
163
- means: "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $."
164
- - key: ar_aged_90_plus
165
- label: "90+ days $"
166
- type: currency
167
- kind: data
168
- means: "Overdue by more than 90 days. The four aging buckets sum to AR overdue $."
169
- - key: days_to_pay
170
- label: "Days to pay"
171
- type: int
172
- kind: data
173
- means: "Average days to pay an invoice in full. Blank means no fully paid invoice yet."
174
- - key: top_category
175
- label: "Top category"
176
- type: text
177
- kind: data
178
- means: "The category this customer spent the most on in the last 12 months."
179
- - key: top_category_pct
180
- label: "Top category %"
181
- type: pct
182
- kind: data
183
- means: "Share of last-12-months spend that went to the top category."
184
- - key: sku_count
185
- label: "SKUs bought"
186
- type: int
187
- kind: data
188
- means: "Distinct products bought in the last 12 months."
189
- - key: top_sku
190
- label: "Top SKU"
191
- type: text
192
- kind: data
193
- means: "The product this customer spent the most on in the last 12 months."
194
- - key: days_since
195
- label: "Days since order"
196
- type: int
197
- kind: data
198
- means: "Days since the last confirmed order."
199
- - key: typical_gap_days
200
- label: "Typical gap days"
201
- type: int
202
- kind: data
203
- means: "Days this customer usually goes between orders, from their own history."
204
- - key: notes
205
- label: "Notes"
206
- type: text
207
- kind: data
208
- means: "Your notes on this customer. Saved in this app only, visible only to you."
209
-
210
- ai_context: >
211
- The customer REGISTRY as a person sees it in the app — the database labelled 'Odoo customers'.
212
- Use it to answer 'who is this customer', 'which customers exist', 'which agent owns them', 'where are they', and for the AR columns it carries (ar_open, ar_overdue, days_to_pay), which are reconciled by modules/ar.py.
213
- ⛔ For anything WINDOWED or at line grain go to sales_lines / customer_invoices / receivables — those topics own the time dimension and this one has none.
 
1
+ # ⭐⭐ W33-T49 (owner item 13) — an ENTITY topic: the semantic layer can finally see
2
+ # the databases a person actually opens, not only the line and document grains.
3
+ #
4
+ # ⛔ THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
5
+ # and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
6
+ # field row: add the column to the grid contract and re-emit, or the agent is being
7
+ # trained on a schema the product does not have.
8
+ key: odoo_customers
9
+ label: "Odoo customers"
10
+ entity: res.partner
11
+ # The database this topic DESCRIBES — the same store key the nav opens (W33-T46).
12
+ grid: customer_data
13
+ subject: "odoo:res.partner"
14
+ grain: "one row per customer in tenant #0's scoped book — a REGISTRY, not a dated event stream. Identity is the Odoo `res.partner` id (the row's `pid` IS that id)."
15
+ scope:
16
+ population: "the customer book `modules/customer_data.pool()` builds; the grid's own contract states it as one row per customer who ordered in the last 24 months"
17
+ identity: "the Odoo res.partner id — exposed as the `partner_id` column (W33-T43) and used as the row pid, so there is ONE id per row and no second one"
18
+ merged: "W33-T44 retired `ut_odoo_customers`, which presented this same subject. This topic describes the SURVIVING database, `customer_data`, which keeps its store bucket so no saved view, grant, cohort or formula moved"
19
+ no_date: "a registry has no date dimension; ask sales_lines or customer_invoices for anything time-windowed about a customer"
20
+ store:
21
+ table: res_partner
22
+ alias: p
23
+ # NO date_col — a registry is not a dated event stream. Stated rather than
24
+ # omitted, so its absence reads as a fact and not as an unfinished file.
25
+ dims:
26
+ agent: {label: "Sales agent"}
27
+ state: {label: "State"}
28
+ country: {label: "Country"}
29
+
30
+ # key / label / type / kind, derived from the grid contract. `kind` says where the
31
+ # value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
32
+ # from another topic; a `link` points at another database.
33
+ fields:
34
+ - key: customer
35
+ label: "Customer"
36
+ type: text
37
+ kind: data
38
+ means: "The customer's name in Odoo. One row per customer who ordered in the last 24 months."
39
+ - key: partner_id
40
+ label: "Odoo ID"
41
+ type: int
42
+ kind: data
43
+ means: "The Odoo res.partner id — this row's identity, and the key every Odoo document joins on. W33-T43 / owner item 12: 'One unique ID per database always.' It is DERIVED rather than read off the pool row because a customer row's pid IS the partner id, so the value is already on every row and a second copy in the pool would be a second source for one fact."
44
+ - key: odoo_status
45
+ label: "Odoo record"
46
+ type: status
47
+ kind: data
48
+ means: "Whether this customer still exists in Odoo. Archived means deleted there."
49
+ - key: agent
50
+ label: "Agent"
51
+ type: text
52
+ kind: data
53
+ means: "The sales agent who owns this account."
54
+ - key: dba
55
+ label: "DBA"
56
+ type: select
57
+ kind: data
58
+ means: "The brand this customer buys from - Fisch, Royal, or both. Amazon-channel orders are not a DBA."
59
+ - key: salesperson
60
+ label: "Salesperson"
61
+ type: text
62
+ kind: data
63
+ means: "Who keyed in most of this customer's orders — not the Agent, who owns the account."
64
+ - key: street
65
+ label: "Street"
66
+ type: text
67
+ kind: data
68
+ means: "First address line, from res.partner directly - not the geocoder, so a customer the map cannot place still shows its address."
69
+ - key: street2
70
+ label: "Street 2"
71
+ type: text
72
+ kind: data
73
+ means: "Second address line (suite, unit, floor) on the customer's Odoo address."
74
+ - key: city
75
+ label: "City"
76
+ type: text
77
+ kind: data
78
+ means: "City on the customer's Odoo address."
79
+ - key: state
80
+ label: "State"
81
+ type: text
82
+ kind: data
83
+ means: "State or province on the customer's Odoo address."
84
+ - key: country
85
+ label: "Country"
86
+ type: text
87
+ kind: data
88
+ means: "Country on the customer's Odoo address."
89
+ - key: zip
90
+ label: "ZIP"
91
+ type: text
92
+ kind: data
93
+ means: "Postal code on the customer's Odoo address."
94
+ - key: customer_since
95
+ label: "Customer since"
96
+ type: date
97
+ kind: data
98
+ means: "When this customer was first set up in Odoo."
99
+ - key: tags
100
+ label: "Tags"
101
+ type: text
102
+ kind: data
103
+ means: "Odoo labels on this customer, comma-separated."
104
+ - key: pricelist
105
+ label: "Price list"
106
+ type: text
107
+ kind: data
108
+ means: "The price list this customer buys on."
109
+ - key: payment_terms
110
+ label: "Payment terms"
111
+ type: text
112
+ kind: data
113
+ means: "Payment terms on this customer's account — Net 30, for example."
114
+ - key: last_order
115
+ label: "Last order"
116
+ type: date
117
+ kind: data
118
+ means: "Date of the most recent confirmed order."
119
+ - key: overdue_days
120
+ label: "Overdue days"
121
+ type: int
122
+ kind: data
123
+ means: "How many days late this customer is running against their own usual ordering rhythm."
124
+ - key: est_missed
125
+ label: "Est. missed $"
126
+ type: currency
127
+ kind: data
128
+ means: "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed."
129
+ - key: ar_open
130
+ label: "AR current $"
131
+ type: currency
132
+ kind: data
133
+ means: "Invoiced money owed but not yet due (a 5-day grace applies before it counts as overdue)."
134
+ - key: ar_overdue
135
+ label: "AR overdue $"
136
+ type: currency
137
+ kind: data
138
+ means: "Invoiced money past due — same basis as the Collections page."
139
+ - key: ar_outstanding
140
+ label: "AR outstanding $"
141
+ type: currency
142
+ kind: data
143
+ means: "Total invoiced money owed right now: AR current $ plus AR overdue $."
144
+ - key: ar_exposure
145
+ label: "Credit exposure $"
146
+ type: currency
147
+ kind: data
148
+ means: "The most you could be out if they stopped paying today: open, overdue, draft and not-yet-invoiced."
149
+ - key: ar_aged_1_30
150
+ label: "1-30 days $"
151
+ type: currency
152
+ kind: data
153
+ means: "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $."
154
+ - key: ar_aged_31_60
155
+ label: "31-60 days $"
156
+ type: currency
157
+ kind: data
158
+ means: "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $."
159
+ - key: ar_aged_61_90
160
+ label: "61-90 days $"
161
+ type: currency
162
+ kind: data
163
+ means: "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $."
164
+ - key: ar_aged_90_plus
165
+ label: "90+ days $"
166
+ type: currency
167
+ kind: data
168
+ means: "Overdue by more than 90 days. The four aging buckets sum to AR overdue $."
169
+ - key: days_to_pay
170
+ label: "Days to pay"
171
+ type: int
172
+ kind: data
173
+ means: "Average days to pay an invoice in full. Blank means no fully paid invoice yet."
174
+ - key: top_category
175
+ label: "Top category"
176
+ type: text
177
+ kind: data
178
+ means: "The category this customer spent the most on in the last 12 months."
179
+ - key: top_category_pct
180
+ label: "Top category %"
181
+ type: pct
182
+ kind: data
183
+ means: "Share of last-12-months spend that went to the top category."
184
+ - key: sku_count
185
+ label: "SKUs bought"
186
+ type: int
187
+ kind: data
188
+ means: "Distinct products bought in the last 12 months."
189
+ - key: top_sku
190
+ label: "Top SKU"
191
+ type: text
192
+ kind: data
193
+ means: "The product this customer spent the most on in the last 12 months."
194
+ - key: days_since
195
+ label: "Days since order"
196
+ type: int
197
+ kind: data
198
+ means: "Days since the last confirmed order."
199
+ - key: typical_gap_days
200
+ label: "Typical gap days"
201
+ type: int
202
+ kind: data
203
+ means: "Days this customer usually goes between orders, from their own history."
204
+ - key: notes
205
+ label: "Notes"
206
+ type: text
207
+ kind: data
208
+ means: "Your notes on this customer. Saved in this app only, visible only to you."
209
+
210
+ ai_context: >
211
+ The customer REGISTRY as a person sees it in the app — the database labelled 'Odoo customers'.
212
+ Use it to answer 'who is this customer', 'which customers exist', 'which agent owns them', 'where are they', and for the AR columns it carries (ar_open, ar_overdue, days_to_pay), which are reconciled by modules/ar.py.
213
+ ⛔ For anything WINDOWED or at line grain go to sales_lines / customer_invoices / receivables — those topics own the time dimension and this one has none.
platform/model/topics/odoo_invoices.yml CHANGED
@@ -1,98 +1,98 @@
1
- # ⭐⭐ W33-T49 (owner item 13) — an ENTITY topic: the semantic layer can finally see
2
- # the databases a person actually opens, not only the line and document grains.
3
- #
4
- # ⛔ THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
5
- # and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
6
- # field row: add the column to the grid contract and re-emit, or the agent is being
7
- # trained on a schema the product does not have.
8
- key: odoo_invoices
9
- label: "Odoo invoices"
10
- entity: account.move
11
- # The database this topic DESCRIBES — the same store key the nav opens (W33-T46).
12
- grid: ut_odoo_invoices
13
- subject: "odoo:account.move.customer"
14
- grain: "one row per posted CUSTOMER invoice or credit note (document grain)"
15
- scope:
16
- population: "move_type in (out_invoice, out_refund) AND state = posted"
17
- signs: "amount_residual_signed is POSITIVE for an invoice and NEGATIVE for a credit note, so summing it nets the credit notes correctly"
18
- store:
19
- table: account_move
20
- alias: m
21
- date_col: "m.invoice_date"
22
- dims:
23
- customer: {label: "Customer"}
24
- payment_state: {label: "Payment state"}
25
- move_type: {label: "Document type"}
26
-
27
- # key / label / type / kind, derived from the grid contract. `kind` says where the
28
- # value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
29
- # from another topic; a `link` points at another database.
30
- fields:
31
- - key: invoice_no
32
- label: "Invoice"
33
- type: text
34
- kind: data
35
- - key: odoo_id
36
- label: "Odoo ID"
37
- type: int
38
- kind: data
39
- means: "The `account.move` id. Also this row's id."
40
- - key: customer
41
- label: "Customer"
42
- type: text
43
- kind: data
44
- - key: partner_id
45
- label: "Odoo partner id"
46
- type: int
47
- kind: data
48
- - key: invoice_date
49
- label: "Invoice date"
50
- type: date
51
- kind: data
52
- - key: due_date
53
- label: "Due date"
54
- type: date
55
- kind: data
56
- - key: residual
57
- label: "Outstanding $"
58
- type: currency
59
- kind: data
60
- means: "Odoo's signed residual. Exactly 0 on every settled document, which is why AR rollups need no filter."
61
- - key: amount_untaxed
62
- label: "Invoiced $"
63
- type: currency
64
- kind: data
65
- - key: payment_state
66
- label: "Payment state"
67
- type: select
68
- kind: data
69
- - key: move_type
70
- label: "Document"
71
- type: select
72
- kind: data
73
- - key: wholesale_scope
74
- label: "In wholesale scope"
75
- type: checkbox
76
- kind: data
77
- means: "Unticked = the GIFTWARE DEALS / Amazon channel, which every wholesale metric in this product excludes. The row is kept so no Odoo id is missing; filter on this column to reconcile against the AR page."
78
- - key: origin_order
79
- label: "Source order"
80
- type: text
81
- kind: data
82
- means: "Odoo's `invoice_origin` - usually the order name, sometimes blank."
83
- - key: order_link
84
- label: "Order record"
85
- type: link
86
- kind: link
87
- to_grid: ut_odoo_orders
88
- - key: refreshed
89
- label: "Refreshed"
90
- type: date
91
- kind: data
92
- means: "When this row was last reconciled against Odoo."
93
-
94
- ai_context: >
95
- Customer invoices at DOCUMENT grain — one row per invoice, not per line.
96
- Use it for AR questions: what is open, what is overdue, what was invoiced.
97
- For anything at product or line grain use invoice_lines; for the reconciled receivables view use receivables.
98
- `origin_order` is the single stored key that joins an invoice back to the order it was raised from.
 
1
+ # ⭐⭐ W33-T49 (owner item 13) — an ENTITY topic: the semantic layer can finally see
2
+ # the databases a person actually opens, not only the line and document grains.
3
+ #
4
+ # ⛔ THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
5
+ # and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
6
+ # field row: add the column to the grid contract and re-emit, or the agent is being
7
+ # trained on a schema the product does not have.
8
+ key: odoo_invoices
9
+ label: "Odoo invoices"
10
+ entity: account.move
11
+ # The database this topic DESCRIBES — the same store key the nav opens (W33-T46).
12
+ grid: ut_odoo_invoices
13
+ subject: "odoo:account.move.customer"
14
+ grain: "one row per posted CUSTOMER invoice or credit note (document grain)"
15
+ scope:
16
+ population: "move_type in (out_invoice, out_refund) AND state = posted"
17
+ signs: "amount_residual_signed is POSITIVE for an invoice and NEGATIVE for a credit note, so summing it nets the credit notes correctly"
18
+ store:
19
+ table: account_move
20
+ alias: m
21
+ date_col: "m.invoice_date"
22
+ dims:
23
+ customer: {label: "Customer"}
24
+ payment_state: {label: "Payment state"}
25
+ move_type: {label: "Document type"}
26
+
27
+ # key / label / type / kind, derived from the grid contract. `kind` says where the
28
+ # value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
29
+ # from another topic; a `link` points at another database.
30
+ fields:
31
+ - key: invoice_no
32
+ label: "Invoice"
33
+ type: text
34
+ kind: data
35
+ - key: odoo_id
36
+ label: "Odoo ID"
37
+ type: int
38
+ kind: data
39
+ means: "The `account.move` id. Also this row's id."
40
+ - key: customer
41
+ label: "Customer"
42
+ type: text
43
+ kind: data
44
+ - key: partner_id
45
+ label: "Odoo partner id"
46
+ type: int
47
+ kind: data
48
+ - key: invoice_date
49
+ label: "Invoice date"
50
+ type: date
51
+ kind: data
52
+ - key: due_date
53
+ label: "Due date"
54
+ type: date
55
+ kind: data
56
+ - key: residual
57
+ label: "Outstanding $"
58
+ type: currency
59
+ kind: data
60
+ means: "Odoo's signed residual. Exactly 0 on every settled document, which is why AR rollups need no filter."
61
+ - key: amount_untaxed
62
+ label: "Invoiced $"
63
+ type: currency
64
+ kind: data
65
+ - key: payment_state
66
+ label: "Payment state"
67
+ type: select
68
+ kind: data
69
+ - key: move_type
70
+ label: "Document"
71
+ type: select
72
+ kind: data
73
+ - key: wholesale_scope
74
+ label: "In wholesale scope"
75
+ type: checkbox
76
+ kind: data
77
+ means: "Unticked = the GIFTWARE DEALS / Amazon channel, which every wholesale metric in this product excludes. The row is kept so no Odoo id is missing; filter on this column to reconcile against the AR page."
78
+ - key: origin_order
79
+ label: "Source order"
80
+ type: text
81
+ kind: data
82
+ means: "Odoo's `invoice_origin` - usually the order name, sometimes blank."
83
+ - key: order_link
84
+ label: "Order record"
85
+ type: link
86
+ kind: link
87
+ to_grid: ut_odoo_orders
88
+ - key: refreshed
89
+ label: "Refreshed"
90
+ type: date
91
+ kind: data
92
+ means: "When this row was last reconciled against Odoo."
93
+
94
+ ai_context: >
95
+ Customer invoices at DOCUMENT grain — one row per invoice, not per line.
96
+ Use it for AR questions: what is open, what is overdue, what was invoiced.
97
+ For anything at product or line grain use invoice_lines; for the reconciled receivables view use receivables.
98
+ `origin_order` is the single stored key that joins an invoice back to the order it was raised from.
platform/model/topics/odoo_orders.yml CHANGED
@@ -1,87 +1,87 @@
1
- # ⭐⭐ W33-T49 (owner item 13) — an ENTITY topic: the semantic layer can finally see
2
- # the databases a person actually opens, not only the line and document grains.
3
- #
4
- # ⛔ THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
5
- # and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
6
- # field row: add the column to the grid contract and re-emit, or the agent is being
7
- # trained on a schema the product does not have.
8
- key: odoo_orders
9
- label: "Odoo orders"
10
- entity: sale.order
11
- # The database this topic DESCRIBES — the same store key the nav opens (W33-T46).
12
- grid: ut_odoo_orders
13
- subject: "odoo:sale.order"
14
- grain: "one row per confirmed sales order (document grain)"
15
- scope:
16
- population: "state in (sale, done) — quotations and cancellations excluded"
17
- vs_sales_orders: "⚠ DISTINCT from the `sales_orders` topic: that one is the wholesale-scoped analytical view (teams 5/6, house accounts excluded); THIS one is the registry a person opens as a grid, and carries the row set the app shows"
18
- store:
19
- table: sale_order
20
- alias: o
21
- date_col: "o.date_order"
22
- dims:
23
- customer: {label: "Customer"}
24
- team: {label: "Business unit"}
25
- state: {label: "State"}
26
- invoice_status: {label: "Invoice status"}
27
-
28
- # key / label / type / kind, derived from the grid contract. `kind` says where the
29
- # value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
30
- # from another topic; a `link` points at another database.
31
- fields:
32
- - key: order_no
33
- label: "Order"
34
- type: text
35
- kind: data
36
- - key: odoo_id
37
- label: "Odoo ID"
38
- type: int
39
- kind: data
40
- means: "The `sale.order` id. Also this row's id."
41
- - key: customer
42
- label: "Customer"
43
- type: text
44
- kind: data
45
- - key: partner_id
46
- label: "Odoo partner id"
47
- type: int
48
- kind: data
49
- - key: order_date
50
- label: "Order date"
51
- type: date
52
- kind: data
53
- - key: amount_untaxed
54
- label: "Order $"
55
- type: currency
56
- kind: data
57
- - key: team
58
- label: "Business unit"
59
- type: select
60
- kind: data
61
- - key: state
62
- label: "State"
63
- type: select
64
- kind: data
65
- - key: invoice_status
66
- label: "Invoice status"
67
- type: select
68
- kind: data
69
- - key: wholesale_scope
70
- label: "In wholesale scope"
71
- type: checkbox
72
- kind: data
73
- means: "Unticked = the GIFTWARE DEALS / Amazon channel, which every wholesale metric in this product excludes. The row is kept so no Odoo id is missing; filter on this column to reconcile against the AR page."
74
- - key: invoices
75
- label: "Invoices"
76
- type: link
77
- kind: link
78
- to_grid: ut_odoo_invoices
79
- - key: refreshed
80
- label: "Refreshed"
81
- type: date
82
- kind: data
83
- means: "When this row was last reconciled against Odoo."
84
-
85
- ai_context: >
86
- Sales orders at DOCUMENT grain, as the app's grid shows them.
87
- ⛔ If you are asked for revenue, prefer `sales_lines` (line grain, wholesale scope) or `sales_orders` — this topic exists to describe the ORDER RECORD a person can open, and its scope is the grid's, not the analytical model's.
 
1
+ # ⭐⭐ W33-T49 (owner item 13) — an ENTITY topic: the semantic layer can finally see
2
+ # the databases a person actually opens, not only the line and document grains.
3
+ #
4
+ # ⛔ THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
5
+ # and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
6
+ # field row: add the column to the grid contract and re-emit, or the agent is being
7
+ # trained on a schema the product does not have.
8
+ key: odoo_orders
9
+ label: "Odoo orders"
10
+ entity: sale.order
11
+ # The database this topic DESCRIBES — the same store key the nav opens (W33-T46).
12
+ grid: ut_odoo_orders
13
+ subject: "odoo:sale.order"
14
+ grain: "one row per confirmed sales order (document grain)"
15
+ scope:
16
+ population: "state in (sale, done) — quotations and cancellations excluded"
17
+ vs_sales_orders: "⚠ DISTINCT from the `sales_orders` topic: that one is the wholesale-scoped analytical view (teams 5/6, house accounts excluded); THIS one is the registry a person opens as a grid, and carries the row set the app shows"
18
+ store:
19
+ table: sale_order
20
+ alias: o
21
+ date_col: "o.date_order"
22
+ dims:
23
+ customer: {label: "Customer"}
24
+ team: {label: "Business unit"}
25
+ state: {label: "State"}
26
+ invoice_status: {label: "Invoice status"}
27
+
28
+ # key / label / type / kind, derived from the grid contract. `kind` says where the
29
+ # value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
30
+ # from another topic; a `link` points at another database.
31
+ fields:
32
+ - key: order_no
33
+ label: "Order"
34
+ type: text
35
+ kind: data
36
+ - key: odoo_id
37
+ label: "Odoo ID"
38
+ type: int
39
+ kind: data
40
+ means: "The `sale.order` id. Also this row's id."
41
+ - key: customer
42
+ label: "Customer"
43
+ type: text
44
+ kind: data
45
+ - key: partner_id
46
+ label: "Odoo partner id"
47
+ type: int
48
+ kind: data
49
+ - key: order_date
50
+ label: "Order date"
51
+ type: date
52
+ kind: data
53
+ - key: amount_untaxed
54
+ label: "Order $"
55
+ type: currency
56
+ kind: data
57
+ - key: team
58
+ label: "Business unit"
59
+ type: select
60
+ kind: data
61
+ - key: state
62
+ label: "State"
63
+ type: select
64
+ kind: data
65
+ - key: invoice_status
66
+ label: "Invoice status"
67
+ type: select
68
+ kind: data
69
+ - key: wholesale_scope
70
+ label: "In wholesale scope"
71
+ type: checkbox
72
+ kind: data
73
+ means: "Unticked = the GIFTWARE DEALS / Amazon channel, which every wholesale metric in this product excludes. The row is kept so no Odoo id is missing; filter on this column to reconcile against the AR page."
74
+ - key: invoices
75
+ label: "Invoices"
76
+ type: link
77
+ kind: link
78
+ to_grid: ut_odoo_invoices
79
+ - key: refreshed
80
+ label: "Refreshed"
81
+ type: date
82
+ kind: data
83
+ means: "When this row was last reconciled against Odoo."
84
+
85
+ ai_context: >
86
+ Sales orders at DOCUMENT grain, as the app's grid shows them.
87
+ ⛔ If you are asked for revenue, prefer `sales_lines` (line grain, wholesale scope) or `sales_orders` — this topic exists to describe the ORDER RECORD a person can open, and its scope is the grid's, not the analytical model's.
platform/model/topics/odoo_products.yml CHANGED
@@ -1,198 +1,198 @@
1
- # ⭐⭐ W33-T49 (owner item 13) — an ENTITY topic: the semantic layer can finally see
2
- # the databases a person actually opens, not only the line and document grains.
3
- #
4
- # ⛔ THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
5
- # and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
6
- # field row: add the column to the grid contract and re-emit, or the agent is being
7
- # trained on a schema the product does not have.
8
- key: odoo_products
9
- label: "Odoo products"
10
- entity: product.product
11
- # The database this topic DESCRIBES — the same store key the nav opens (W33-T46).
12
- grid: product_data
13
- subject: "odoo:product.product"
14
- grain: "one row per SKU in the active catalogue — a CATALOGUE, not a dated event stream"
15
- scope:
16
- population: "every ACTIVE product, sold or not (measured 5,862). Deliberately not the sales-window universe: that lands at 2,717 and hides ~2,550 SKUs that have never sold in wholesale scope"
17
- identity: "the SKU code (`default_code`). ⚠ 33 active products carry NO code and are keyed `pid:<odoo product id>` instead, so every active product has exactly one row and none is dropped"
18
- merged: "W33-T44 retired `ut_odoo_products`, which presented this same subject on Odoo's product_id. This topic describes the SURVIVING database, `product_data`"
19
- no_date: "a catalogue has no date dimension; ask sales_lines for movement"
20
- store:
21
- table: product_product
22
- alias: pp
23
- # NO date_col — a registry is not a dated event stream. Stated rather than
24
- # omitted, so its absence reads as a fact and not as an unfinished file.
25
- dims:
26
- category: {label: "Category"}
27
- supplier: {label: "Supplier"}
28
-
29
- # ⭐⭐ W37-T10 / owner item 4 (ruling R1) — THE LOOKBACK-MEASURE BINDING. Owner: *"Odoo products
30
- # should have a lookbsck metrics like sales etc."* This block is what turns `measures: []` on the
31
- # product workspace into a real catalogue, and it is DECLARED HERE rather than hand-written in a
32
- # route for the reason `_rollup_source_offer` gives about itself: a second list is a second
33
- # definition of the same fact, and the two drift the day somebody adds a metric.
34
- #
35
- # ⛔ THE JOIN KEY IS THE TRAP CONTRACT C1 NAMES, and this is where it is answered. `source` is a
36
- # FACT topic; `dim` is the dim of THAT topic which carries THIS entity's identity. `sales_lines`
37
- # has two candidates and only one is right: `product` groups by Odoo's `product_id`, while this
38
- # database's identity (see `scope.identity` above) is `default_code`. Binding to `product` would
39
- # key every cell on a number no grid row carries — a column of nulls that reads as "never sold".
40
- #
41
- # ⚠ A SINGLE-TOPIC GROUPED QUERY, which is why `semantic.store_query`'s cross-topic refusal does
42
- # not bite: `revenue` lives ON `sales_lines` and we group `sales_lines` by its OWN dim. The same
43
- # refusal is real and DOES bite `aov`/`orders` (denominator `orders` lives on `sales_orders`), so
44
- # those two are absent from `keys` below and `entity_measures()` re-proves that rather than
45
- # trusting this list.
46
- # ⭐ A LIST SINCE W37-T13: an entity's columns legitimately come from more than ONE fact topic.
47
- # Sales movement is `sales_lines`; physical movement is `stock_moves`. Same grid, same join key
48
- # (`product_code`), different tables — forcing stock into the sales topic would have defined a
49
- # metric against rows it does not have.
50
- measures:
51
- - source: sales_lines
52
- dim: product_code
53
- # ⭐ The SHIP-FIRST six of `proto/P3-metric-catalog.md`, and they cost ONE grouped query
54
- # together (measured 0.22 s over 1,670 groups against the mirror; 2.55 s live).
55
- keys: [units, revenue, margin, cogs, margin_pct, asp]
56
- # ⛔ REPORTED, NOT SILENTLY DROPPED (standing rule 1's second sentence, applied to a catalogue).
57
- # Each of these is a real metric `proto/P3-metric-catalog.md` measured and this wave does not
58
- # ship, with the CAUSE and the fix — so the next session extends the list instead of
59
- # re-measuring, and nobody reads the six as "all Odoo can answer".
60
- not_yet:
61
- - key: days_since_last_sale
62
- cause: "needs `agg: max` over a DATE, which `semantic._measure_sql` does not implement (sum | count | count_distinct only), and a date-typed measure could not render anyway: `aios_grid.MEASURE_FIELD_TYPES` is {currency, int, pct}"
63
- fix: "add `agg: max` to `_measure_sql` and express the metric as an INT — `date_diff('day', max(date), today)` — so it renders as a number of days rather than a date. Measured live at 4.61 s over 3,348 SKUs"
64
- - key: distinct_orders
65
- cause: "`__count` on the live path is LINE count, not ORDER count (12,707 lines vs 12,645 distinct SKU-order pairs); the honest figure needs a 2-level groupby measured at 10.87 s live"
66
- fix: "on the STORE path this is `count(DISTINCT l.order_id)` in one pass — add it as a `count_distinct` metric on `sales_lines` with field `order_id`"
67
-
68
- # ⭐⭐ W37-T13 — PHYSICAL MOVEMENT. `stock.move` and `stock_location` are mirrored now (the
69
- # `not_yet` entry that used to sit here said "not mirrored, so there is no store binding to
70
- # group"; that is what changed). The direction rule is TWO-SIDED and lives in the metrics'
71
- # `store_filter_sql` — see `model/metrics/stock.yml`, which carries the 58% trap in full.
72
- - source: stock_moves
73
- dim: product_code
74
- keys: [stock_in, stock_out, stock_net]
75
- not_yet:
76
- - key: stock_in_excl_adjustments
77
- cause: "adjustments and scrap are ~30% of IN / ~20% of OUT and the two obvious exclusions are NOT equivalent - the 5,320-unit gap between them is entirely `Virtual Locations/Scrap`, so the choice is a business ruling rather than a filter"
78
- fix: "ask the owner which exclusion they mean, then add it as a SEPARATE metric with `store_filter_sql` narrowed on `sl.usage`/`sd.usage` - never by changing what `stock_in` means"
79
-
80
- # key / label / type / kind, derived from the grid contract. `kind` says where the
81
- # value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
82
- # from another topic; a `link` points at another database.
83
- fields:
84
- - key: code
85
- label: "SKU"
86
- type: text
87
- kind: data
88
- means: "The SKU code — the product's real business key. `pid` is a stable CRC32 of it because the grid keys on an integer."
89
- - key: product
90
- label: "Product"
91
- type: text
92
- kind: data
93
- means: "Product name as it appears in Odoo."
94
- - key: category
95
- label: "Category"
96
- type: select
97
- kind: data
98
- means: "Product category; '(uncategorized)' when Odoo carries none."
99
- - key: supplier
100
- label: "Supplier"
101
- type: text
102
- kind: data
103
- means: "Who makes it. Editable here and shared with everyone in the workspace; seeded from the inventory mastersheet."
104
- - key: origin_country
105
- label: "Country"
106
- type: text
107
- kind: data
108
- means: "Country of origin. Editable here and shared with everyone; seeded from the inventory mastersheet."
109
- - key: lead_days
110
- label: "Lead time (days)"
111
- type: int
112
- kind: data
113
- means: "Order-to-arrival days for this supplier. Drives the buy signal. Editable and shared with everyone."
114
- - key: first_cost
115
- label: "First cost"
116
- type: currency
117
- kind: data
118
- means: "Quoted unit cost at origin, before freight and duty. Editable and shared with everyone."
119
- - key: price_fisch
120
- label: "Fisch price"
121
- type: currency
122
- kind: data
123
- means: "Fisch pricelist price for this SKU. Blank when that list prices it nowhere."
124
- - key: price_royal_1
125
- label: "Royal 1 price"
126
- type: currency
127
- kind: data
128
- means: "Royal 1 pricelist price for this SKU. Blank when that list prices it nowhere."
129
- - key: price_royal_2
130
- label: "Royal 2 price"
131
- type: currency
132
- kind: data
133
- means: "Royal 2 pricelist price for this SKU. Blank when that list prices it nowhere."
134
- - key: rev_ytd
135
- label: "Revenue YTD"
136
- type: currency
137
- kind: data
138
- means: "Year-to-date revenue for this SKU, BU-scoped when the caller is."
139
- - key: rev_ly
140
- label: "Revenue LY"
141
- type: currency
142
- kind: data
143
- means: "Same period last year — seasonal wholesale compares like for like."
144
- - key: yoy_pct
145
- label: "YoY %"
146
- type: pct
147
- kind: data
148
- means: "Year-over-year change; null when last year was zero (a ratio to zero is not a number)."
149
- - key: qty_ytd
150
- label: "Units YTD"
151
- type: int
152
- kind: data
153
- means: "Units sold year to date."
154
- - key: orders_ytd
155
- label: "Orders YTD"
156
- type: int
157
- kind: data
158
- means: "Distinct orders containing this SKU, year to date."
159
- - key: on_hand
160
- label: "On hand"
161
- type: int
162
- kind: data
163
- means: "Units in stock. CONSOLIDATED — one physical warehouse, not brand-tagged, so this column is ABSENT for a BU-scoped caller rather than silently company-wide."
164
- - key: unit_cost
165
- label: "Unit cost"
166
- type: currency
167
- kind: data
168
- means: "Inventory unit cost. Consolidated; absent for a BU-scoped caller."
169
- - key: inv_value
170
- label: "Stock value"
171
- type: currency
172
- kind: data
173
- means: "On-hand value at cost. Consolidated; absent for a BU-scoped caller."
174
- - key: qty_ltm
175
- label: "Units LTM"
176
- type: int
177
- kind: data
178
- means: "Units sold in the last twelve months. Consolidated; absent for a BU-scoped caller."
179
- - key: dos
180
- label: "Days of supply"
181
- type: int
182
- kind: data
183
- means: "Days of supply at the LTM rate; null means it never sells through. Consolidated; absent for a BU-scoped caller."
184
- - key: cover_gap_d
185
- label: "Cover gap (days)"
186
- type: int
187
- kind: data
188
- means: "Days of supply minus lead time. Negative means it runs out before a reorder lands."
189
- - key: stock_bucket
190
- label: "Stock status"
191
- type: select
192
- kind: data
193
- means: "Dead / excess / healthy bucket from the inventory module. Consolidated; absent for a BU-scoped caller."
194
-
195
- ai_context: >
196
- The product CATALOGUE — the database labelled 'Odoo products'.
197
- Use it for 'what do we sell', 'what does it cost' (first_cost/unit_cost), 'who supplies it', 'what is on hand' and the stock-coverage columns.
198
- ⛔ Revenue and unit columns here are WINDOWED SNAPSHOTS (rev_ytd, qty_ltm); for any other window, or for line grain, go to sales_lines.
 
1
+ # ⭐⭐ W33-T49 (owner item 13) — an ENTITY topic: the semantic layer can finally see
2
+ # the databases a person actually opens, not only the line and document grains.
3
+ #
4
+ # ⛔ THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
5
+ # and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
6
+ # field row: add the column to the grid contract and re-emit, or the agent is being
7
+ # trained on a schema the product does not have.
8
+ key: odoo_products
9
+ label: "Odoo products"
10
+ entity: product.product
11
+ # The database this topic DESCRIBES — the same store key the nav opens (W33-T46).
12
+ grid: product_data
13
+ subject: "odoo:product.product"
14
+ grain: "one row per SKU in the active catalogue — a CATALOGUE, not a dated event stream"
15
+ scope:
16
+ population: "every ACTIVE product, sold or not (measured 5,862). Deliberately not the sales-window universe: that lands at 2,717 and hides ~2,550 SKUs that have never sold in wholesale scope"
17
+ identity: "the SKU code (`default_code`). ⚠ 33 active products carry NO code and are keyed `pid:<odoo product id>` instead, so every active product has exactly one row and none is dropped"
18
+ merged: "W33-T44 retired `ut_odoo_products`, which presented this same subject on Odoo's product_id. This topic describes the SURVIVING database, `product_data`"
19
+ no_date: "a catalogue has no date dimension; ask sales_lines for movement"
20
+ store:
21
+ table: product_product
22
+ alias: pp
23
+ # NO date_col — a registry is not a dated event stream. Stated rather than
24
+ # omitted, so its absence reads as a fact and not as an unfinished file.
25
+ dims:
26
+ category: {label: "Category"}
27
+ supplier: {label: "Supplier"}
28
+
29
+ # ⭐⭐ W37-T10 / owner item 4 (ruling R1) — THE LOOKBACK-MEASURE BINDING. Owner: *"Odoo products
30
+ # should have a lookbsck metrics like sales etc."* This block is what turns `measures: []` on the
31
+ # product workspace into a real catalogue, and it is DECLARED HERE rather than hand-written in a
32
+ # route for the reason `_rollup_source_offer` gives about itself: a second list is a second
33
+ # definition of the same fact, and the two drift the day somebody adds a metric.
34
+ #
35
+ # ⛔ THE JOIN KEY IS THE TRAP CONTRACT C1 NAMES, and this is where it is answered. `source` is a
36
+ # FACT topic; `dim` is the dim of THAT topic which carries THIS entity's identity. `sales_lines`
37
+ # has two candidates and only one is right: `product` groups by Odoo's `product_id`, while this
38
+ # database's identity (see `scope.identity` above) is `default_code`. Binding to `product` would
39
+ # key every cell on a number no grid row carries — a column of nulls that reads as "never sold".
40
+ #
41
+ # ⚠ A SINGLE-TOPIC GROUPED QUERY, which is why `semantic.store_query`'s cross-topic refusal does
42
+ # not bite: `revenue` lives ON `sales_lines` and we group `sales_lines` by its OWN dim. The same
43
+ # refusal is real and DOES bite `aov`/`orders` (denominator `orders` lives on `sales_orders`), so
44
+ # those two are absent from `keys` below and `entity_measures()` re-proves that rather than
45
+ # trusting this list.
46
+ # ⭐ A LIST SINCE W37-T13: an entity's columns legitimately come from more than ONE fact topic.
47
+ # Sales movement is `sales_lines`; physical movement is `stock_moves`. Same grid, same join key
48
+ # (`product_code`), different tables — forcing stock into the sales topic would have defined a
49
+ # metric against rows it does not have.
50
+ measures:
51
+ - source: sales_lines
52
+ dim: product_code
53
+ # ⭐ The SHIP-FIRST six of `proto/P3-metric-catalog.md`, and they cost ONE grouped query
54
+ # together (measured 0.22 s over 1,670 groups against the mirror; 2.55 s live).
55
+ keys: [units, revenue, margin, cogs, margin_pct, asp]
56
+ # ⛔ REPORTED, NOT SILENTLY DROPPED (standing rule 1's second sentence, applied to a catalogue).
57
+ # Each of these is a real metric `proto/P3-metric-catalog.md` measured and this wave does not
58
+ # ship, with the CAUSE and the fix — so the next session extends the list instead of
59
+ # re-measuring, and nobody reads the six as "all Odoo can answer".
60
+ not_yet:
61
+ - key: days_since_last_sale
62
+ cause: "needs `agg: max` over a DATE, which `semantic._measure_sql` does not implement (sum | count | count_distinct only), and a date-typed measure could not render anyway: `aios_grid.MEASURE_FIELD_TYPES` is {currency, int, pct}"
63
+ fix: "add `agg: max` to `_measure_sql` and express the metric as an INT — `date_diff('day', max(date), today)` — so it renders as a number of days rather than a date. Measured live at 4.61 s over 3,348 SKUs"
64
+ - key: distinct_orders
65
+ cause: "`__count` on the live path is LINE count, not ORDER count (12,707 lines vs 12,645 distinct SKU-order pairs); the honest figure needs a 2-level groupby measured at 10.87 s live"
66
+ fix: "on the STORE path this is `count(DISTINCT l.order_id)` in one pass — add it as a `count_distinct` metric on `sales_lines` with field `order_id`"
67
+
68
+ # ⭐⭐ W37-T13 — PHYSICAL MOVEMENT. `stock.move` and `stock_location` are mirrored now (the
69
+ # `not_yet` entry that used to sit here said "not mirrored, so there is no store binding to
70
+ # group"; that is what changed). The direction rule is TWO-SIDED and lives in the metrics'
71
+ # `store_filter_sql` — see `model/metrics/stock.yml`, which carries the 58% trap in full.
72
+ - source: stock_moves
73
+ dim: product_code
74
+ keys: [stock_in, stock_out, stock_net]
75
+ not_yet:
76
+ - key: stock_in_excl_adjustments
77
+ cause: "adjustments and scrap are ~30% of IN / ~20% of OUT and the two obvious exclusions are NOT equivalent - the 5,320-unit gap between them is entirely `Virtual Locations/Scrap`, so the choice is a business ruling rather than a filter"
78
+ fix: "ask the owner which exclusion they mean, then add it as a SEPARATE metric with `store_filter_sql` narrowed on `sl.usage`/`sd.usage` - never by changing what `stock_in` means"
79
+
80
+ # key / label / type / kind, derived from the grid contract. `kind` says where the
81
+ # value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
82
+ # from another topic; a `link` points at another database.
83
+ fields:
84
+ - key: code
85
+ label: "SKU"
86
+ type: text
87
+ kind: data
88
+ means: "The SKU code — the product's real business key. `pid` is a stable CRC32 of it because the grid keys on an integer."
89
+ - key: product
90
+ label: "Product"
91
+ type: text
92
+ kind: data
93
+ means: "Product name as it appears in Odoo."
94
+ - key: category
95
+ label: "Category"
96
+ type: select
97
+ kind: data
98
+ means: "Product category; '(uncategorized)' when Odoo carries none."
99
+ - key: supplier
100
+ label: "Supplier"
101
+ type: text
102
+ kind: data
103
+ means: "Who makes it. Editable here and shared with everyone in the workspace; seeded from the inventory mastersheet."
104
+ - key: origin_country
105
+ label: "Country"
106
+ type: text
107
+ kind: data
108
+ means: "Country of origin. Editable here and shared with everyone; seeded from the inventory mastersheet."
109
+ - key: lead_days
110
+ label: "Lead time (days)"
111
+ type: int
112
+ kind: data
113
+ means: "Order-to-arrival days for this supplier. Drives the buy signal. Editable and shared with everyone."
114
+ - key: first_cost
115
+ label: "First cost"
116
+ type: currency
117
+ kind: data
118
+ means: "Quoted unit cost at origin, before freight and duty. Editable and shared with everyone."
119
+ - key: price_fisch
120
+ label: "Fisch price"
121
+ type: currency
122
+ kind: data
123
+ means: "Fisch pricelist price for this SKU. Blank when that list prices it nowhere."
124
+ - key: price_royal_1
125
+ label: "Royal 1 price"
126
+ type: currency
127
+ kind: data
128
+ means: "Royal 1 pricelist price for this SKU. Blank when that list prices it nowhere."
129
+ - key: price_royal_2
130
+ label: "Royal 2 price"
131
+ type: currency
132
+ kind: data
133
+ means: "Royal 2 pricelist price for this SKU. Blank when that list prices it nowhere."
134
+ - key: rev_ytd
135
+ label: "Revenue YTD"
136
+ type: currency
137
+ kind: data
138
+ means: "Year-to-date revenue for this SKU, BU-scoped when the caller is."
139
+ - key: rev_ly
140
+ label: "Revenue LY"
141
+ type: currency
142
+ kind: data
143
+ means: "Same period last year — seasonal wholesale compares like for like."
144
+ - key: yoy_pct
145
+ label: "YoY %"
146
+ type: pct
147
+ kind: data
148
+ means: "Year-over-year change; null when last year was zero (a ratio to zero is not a number)."
149
+ - key: qty_ytd
150
+ label: "Units YTD"
151
+ type: int
152
+ kind: data
153
+ means: "Units sold year to date."
154
+ - key: orders_ytd
155
+ label: "Orders YTD"
156
+ type: int
157
+ kind: data
158
+ means: "Distinct orders containing this SKU, year to date."
159
+ - key: on_hand
160
+ label: "On hand"
161
+ type: int
162
+ kind: data
163
+ means: "Units in stock. CONSOLIDATED — one physical warehouse, not brand-tagged, so this column is ABSENT for a BU-scoped caller rather than silently company-wide."
164
+ - key: unit_cost
165
+ label: "Unit cost"
166
+ type: currency
167
+ kind: data
168
+ means: "Inventory unit cost. Consolidated; absent for a BU-scoped caller."
169
+ - key: inv_value
170
+ label: "Stock value"
171
+ type: currency
172
+ kind: data
173
+ means: "On-hand value at cost. Consolidated; absent for a BU-scoped caller."
174
+ - key: qty_ltm
175
+ label: "Units LTM"
176
+ type: int
177
+ kind: data
178
+ means: "Units sold in the last twelve months. Consolidated; absent for a BU-scoped caller."
179
+ - key: dos
180
+ label: "Days of supply"
181
+ type: int
182
+ kind: data
183
+ means: "Days of supply at the LTM rate; null means it never sells through. Consolidated; absent for a BU-scoped caller."
184
+ - key: cover_gap_d
185
+ label: "Cover gap (days)"
186
+ type: int
187
+ kind: data
188
+ means: "Days of supply minus lead time. Negative means it runs out before a reorder lands."
189
+ - key: stock_bucket
190
+ label: "Stock status"
191
+ type: select
192
+ kind: data
193
+ means: "Dead / excess / healthy bucket from the inventory module. Consolidated; absent for a BU-scoped caller."
194
+
195
+ ai_context: >
196
+ The product CATALOGUE — the database labelled 'Odoo products'.
197
+ Use it for 'what do we sell', 'what does it cost' (first_cost/unit_cost), 'who supplies it', 'what is on hand' and the stock-coverage columns.
198
+ ⛔ Revenue and unit columns here are WINDOWED SNAPSHOTS (rev_ytd, qty_ltm); for any other window, or for line grain, go to sales_lines.
platform/model/topics/odoo_vendors.yml CHANGED
@@ -1,103 +1,103 @@
1
- # ⭐⭐ W33-T49 (owner item 13) — an ENTITY topic: the semantic layer can finally see
2
- # the databases a person actually opens, not only the line and document grains.
3
- #
4
- # ⛔ THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
5
- # and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
6
- # field row: add the column to the grid contract and re-emit, or the agent is being
7
- # trained on a schema the product does not have.
8
- key: odoo_vendors
9
- label: "Odoo vendors"
10
- entity: res.partner
11
- # The database this topic DESCRIBES — the same store key the nav opens (W33-T46).
12
- grid: ut_odoo_vendors
13
- subject: "odoo:res.partner.vendor"
14
- grain: "one row per partner we have posted a vendor bill to"
15
- scope:
16
- population: "DERIVED FROM THE BILLS — a partner with no posted vendor bill has no payable history to show, so the bill is the population"
17
- asymmetry: "deliberately unlike odoo_customers, which is NOT derived from its documents; there is no second document universe for vendors"
18
- overlap: "measured: only 9 of the vendors also appear in the customer population, so a vendor is not a customer row"
19
- store:
20
- table: res_partner
21
- alias: p
22
- # NO date_col — a registry is not a dated event stream. Stated rather than
23
- # omitted, so its absence reads as a fact and not as an unfinished file.
24
- dims:
25
- country: {label: "Country"}
26
-
27
- # key / label / type / kind, derived from the grid contract. `kind` says where the
28
- # value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
29
- # from another topic; a `link` points at another database.
30
- fields:
31
- - key: vendor
32
- label: "Vendor"
33
- type: text
34
- kind: data
35
- - key: odoo_id
36
- label: "Odoo ID"
37
- type: int
38
- kind: data
39
- means: "The `res.partner` id. Also this row's id."
40
- - key: vendor_id
41
- label: "Odoo vendor id"
42
- type: int
43
- kind: data
44
- - key: country
45
- label: "Country"
46
- type: text
47
- kind: data
48
- - key: email
49
- label: "Email"
50
- type: text
51
- kind: data
52
- - key: phone
53
- label: "Phone"
54
- type: text
55
- kind: data
56
- - key: mobile
57
- label: "Mobile"
58
- type: text
59
- kind: data
60
- - key: vat
61
- label: "Tax ID"
62
- type: text
63
- kind: data
64
- means: "Odoo `vat` — the vendor's tax/VAT registration number."
65
- - key: vendor_ref
66
- label: "Vendor reference"
67
- type: text
68
- kind: data
69
- means: "Odoo `res.partner.ref` — our internal reference for this vendor."
70
- - key: website
71
- label: "Website"
72
- type: url
73
- kind: data
74
- - key: street
75
- label: "Street"
76
- type: text
77
- kind: data
78
- - key: street2
79
- label: "Street 2"
80
- type: text
81
- kind: data
82
- - key: city
83
- label: "City"
84
- type: text
85
- kind: data
86
- - key: zip
87
- label: "ZIP"
88
- type: text
89
- kind: data
90
- - key: bills
91
- label: "Bills"
92
- type: link
93
- kind: link
94
- to_grid: ut_odoo_bills
95
- - key: refreshed
96
- label: "Refreshed"
97
- type: date
98
- kind: data
99
- means: "When this row was last reconciled against Odoo."
100
-
101
- ai_context: >
102
- The vendor/supplier registry, with contact and tax identity (W33-T48 widened it from 5 columns to 15 after a census found 76 populated fields on the underlying partner)
103
- ⚠ The contact columns are SPARSE by nature — measured 65/393 with an email, 13/393 with a tax id — so 'blank' means Odoo has no value, never that the sync failed.
 
1
+ # ⭐⭐ W33-T49 (owner item 13) — an ENTITY topic: the semantic layer can finally see
2
+ # the databases a person actually opens, not only the line and document grains.
3
+ #
4
+ # ⛔ THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT
5
+ # and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a
6
+ # field row: add the column to the grid contract and re-emit, or the agent is being
7
+ # trained on a schema the product does not have.
8
+ key: odoo_vendors
9
+ label: "Odoo vendors"
10
+ entity: res.partner
11
+ # The database this topic DESCRIBES — the same store key the nav opens (W33-T46).
12
+ grid: ut_odoo_vendors
13
+ subject: "odoo:res.partner.vendor"
14
+ grain: "one row per partner we have posted a vendor bill to"
15
+ scope:
16
+ population: "DERIVED FROM THE BILLS — a partner with no posted vendor bill has no payable history to show, so the bill is the population"
17
+ asymmetry: "deliberately unlike odoo_customers, which is NOT derived from its documents; there is no second document universe for vendors"
18
+ overlap: "measured: only 9 of the vendors also appear in the customer population, so a vendor is not a customer row"
19
+ store:
20
+ table: res_partner
21
+ alias: p
22
+ # NO date_col — a registry is not a dated event stream. Stated rather than
23
+ # omitted, so its absence reads as a fact and not as an unfinished file.
24
+ dims:
25
+ country: {label: "Country"}
26
+
27
+ # key / label / type / kind, derived from the grid contract. `kind` says where the
28
+ # value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
29
+ # from another topic; a `link` points at another database.
30
+ fields:
31
+ - key: vendor
32
+ label: "Vendor"
33
+ type: text
34
+ kind: data
35
+ - key: odoo_id
36
+ label: "Odoo ID"
37
+ type: int
38
+ kind: data
39
+ means: "The `res.partner` id. Also this row's id."
40
+ - key: vendor_id
41
+ label: "Odoo vendor id"
42
+ type: int
43
+ kind: data
44
+ - key: country
45
+ label: "Country"
46
+ type: text
47
+ kind: data
48
+ - key: email
49
+ label: "Email"
50
+ type: text
51
+ kind: data
52
+ - key: phone
53
+ label: "Phone"
54
+ type: text
55
+ kind: data
56
+ - key: mobile
57
+ label: "Mobile"
58
+ type: text
59
+ kind: data
60
+ - key: vat
61
+ label: "Tax ID"
62
+ type: text
63
+ kind: data
64
+ means: "Odoo `vat` — the vendor's tax/VAT registration number."
65
+ - key: vendor_ref
66
+ label: "Vendor reference"
67
+ type: text
68
+ kind: data
69
+ means: "Odoo `res.partner.ref` — our internal reference for this vendor."
70
+ - key: website
71
+ label: "Website"
72
+ type: url
73
+ kind: data
74
+ - key: street
75
+ label: "Street"
76
+ type: text
77
+ kind: data
78
+ - key: street2
79
+ label: "Street 2"
80
+ type: text
81
+ kind: data
82
+ - key: city
83
+ label: "City"
84
+ type: text
85
+ kind: data
86
+ - key: zip
87
+ label: "ZIP"
88
+ type: text
89
+ kind: data
90
+ - key: bills
91
+ label: "Bills"
92
+ type: link
93
+ kind: link
94
+ to_grid: ut_odoo_bills
95
+ - key: refreshed
96
+ label: "Refreshed"
97
+ type: date
98
+ kind: data
99
+ means: "When this row was last reconciled against Odoo."
100
+
101
+ ai_context: >
102
+ The vendor/supplier registry, with contact and tax identity (W33-T48 widened it from 5 columns to 15 after a census found 76 populated fields on the underlying partner)
103
+ ⚠ The contact columns are SPARSE by nature — measured 65/393 with an email, 13/393 with a tax id — so 'blank' means Odoo has no value, never that the sync failed.
platform/modules/agent.py CHANGED
@@ -1,385 +1,385 @@
1
- """Agent module — per-agent (res.partner.agent_ids) analytics.
2
-
3
- An *agent* owns a **book** of customers (the same attribute the Customers module slices by). This
4
- module reports that book the way Sales/Customers/SKU report the whole company: a period scorecard
5
- with custom date windows (Today / WTD / Last week / MTD / QTD / YTD / any custom range), a sales
6
- trend, returns, top SKUs (with profit/order) and the FULL customer list — INCLUDING inactive
7
- accounts (no recent orders) so a rep sees who they've stopped selling to.
8
-
9
- Scope: reuses the Sales `order_domain` (Fisch+Royal, excluded accounts removed, state sale/done)
10
- so numbers tie to every other module. Returns are consolidated (credit notes aren't BU-tagged);
11
- everything else is BU-filterable via team_id.
12
- """
13
- import sys
14
- import datetime as dt
15
- from pathlib import Path
16
- sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
17
- import core.odoo as O
18
- import core.periods as P
19
- import modules.sales as sales_mod
20
- import modules.customers as cust_mod
21
-
22
-
23
- def options(t=None, team_id=None):
24
- """Agent names with book activity — for the page/drawer picker."""
25
- return cust_mod.agent_options(t, team_id)
26
-
27
-
28
- def _book(name):
29
- """frozenset of every partner id assigned to the agent (incl. inactive). None only for 'All'."""
30
- return cust_mod.agent_partner_ids(name)
31
-
32
-
33
- def _rev(date_from, date_to, team_id, book):
34
- return O.sum_field('sale.order', sales_mod.order_domain(date_from, date_to, team_id, partner_ids=book),
35
- 'amount_untaxed')
36
-
37
-
38
- def _orders(date_from, date_to, team_id, book):
39
- return O.get_odoo().search_count('sale.order', sales_mod.order_domain(date_from, date_to, team_id, partner_ids=book))
40
-
41
-
42
- def _custs(date_from, date_to, team_id, book):
43
- return O.distinct_count('sale.order', sales_mod.order_domain(date_from, date_to, team_id, partner_ids=book),
44
- 'partner_id')
45
-
46
-
47
- # ------------------------------------------------------------ period scorecard (custom dating)
48
- # Today / WTD / Last week / MTD / QTD / YTD — each carries its window so the UI can decompose it
49
- # and so "what did this agent sell this/last week" is a click, not a date-math exercise.
50
- _PERIODS = [('Today', 'today'), ('Week to date', 'wtd'), ('Last week', 'lwk'),
51
- ('Month to date', 'mtd'), ('Quarter to date', 'qtd'), ('Year to date', 'ytd')]
52
-
53
-
54
- def _window(key, t):
55
- if key == 'today':
56
- return P._d(t), P._d(t)
57
- if key == 'lwk': # the full prior Mon–Sun week
58
- f, _tt = P.wtd(t)
59
- start = dt.date.fromisoformat(f) - dt.timedelta(days=7)
60
- return start.isoformat(), (dt.date.fromisoformat(f) - dt.timedelta(days=1)).isoformat()
61
- return {'wtd': P.wtd, 'mtd': P.mtd, 'qtd': P.qtd, 'ytd': P.ytd}[key](t)
62
-
63
-
64
- def scorecard(name, t=None, team_id=None):
65
- """Book revenue (+ YoY same-period), orders for Today / WTD / Last week / MTD / QTD / YTD."""
66
- t = t or P.today()
67
- book = _book(name)
68
- out = []
69
- for label, key in _PERIODS:
70
- f, tt = _window(key, t)
71
- wk = key in ('today', 'wtd', 'lwk') # weekday-align the short windows' LY compare
72
- cf, ct = P.shift_year(f, tt, weeks=wk)
73
- rev, rev_ly = _rev(f, tt, team_id, book), _rev(cf, ct, team_id, book)
74
- out.append({'key': key, 'label': label, 'date_from': f, 'date_to': tt, 'cmp_from': cf, 'cmp_to': ct,
75
- 'revenue': rev, 'revenue_ly': rev_ly, 'yoy_pct': P.yoy_pct(rev, rev_ly),
76
- 'orders': _orders(f, tt, team_id, book)})
77
- return out
78
-
79
-
80
- def headline(name, date_from, date_to, team_id=None):
81
- """Book KPIs for an ARBITRARY window (custom dating): revenue + YoY (same window LY), orders,
82
- active customers, AOV and returns $ / return rate."""
83
- book = _book(name)
84
- cf, ct = P.shift_year(date_from, date_to, weeks=False)
85
- rev, rev_ly = _rev(date_from, date_to, team_id, book), _rev(cf, ct, team_id, book)
86
- orders = _orders(date_from, date_to, team_id, book)
87
- ret = sales_mod._returns_amt(date_from, date_to, book)
88
- return {'date_from': date_from, 'date_to': date_to, 'cmp_from': cf, 'cmp_to': ct,
89
- 'revenue': rev, 'revenue_ly': rev_ly,
90
- 'yoy_pct': P.yoy_pct(rev, rev_ly), 'orders': orders,
91
- 'customers': _custs(date_from, date_to, team_id, book), 'aov': (rev / orders) if orders else 0.0,
92
- 'returns': ret, 'return_rate_pct': (ret / rev * 100.0) if rev else 0.0}
93
-
94
-
95
- # ------------------------------------------------------------ sales trend
96
- def _book_monthly_rev(book, date_from, date_to, team_id=None):
97
- g = O.read_group('sale.order', sales_mod.order_domain(date_from, date_to, team_id, partner_ids=book),
98
- ['amount_untaxed:sum'], ['date_order:month'], lazy=False)
99
- out = {}
100
- for r in g:
101
- ym = ((r.get('__range') or {}).get('date_order:month') or {}).get('from', '')[:7]
102
- if ym:
103
- out[ym] = r.get('amount_untaxed') or 0.0
104
- return out
105
-
106
-
107
- def monthly(name, n=13, t=None, team_id=None):
108
- """Book sales per month vs the same month last year (one month-grouped query over 2 years)."""
109
- t = t or P.today()
110
- book = _book(name)
111
- mrev = _book_monthly_rev(book, dt.date(t.year - 2, t.month, 1).isoformat(), t.isoformat(), team_id)
112
- rows = []
113
- for ym, _s, _e in P.month_starts(n, t):
114
- y, m = int(ym[:4]) - 1, int(ym[5:7])
115
- this, last = mrev.get(ym, 0.0), mrev.get(f'{y:04d}-{m:02d}', 0.0)
116
- rows.append({'month': ym, 'revenue': this, 'revenue_ly': last, 'yoy_pct': P.yoy_pct(this, last)})
117
- return rows
118
-
119
-
120
- # ------------------------------------------------------------ full customer book (incl. inactive)
121
- def customers(name, t=None, team_id=None):
122
- """EVERY customer in the agent's book, including inactive accounts (no YTD/LY orders) — those
123
- show $0 with status 'Inactive'/'Dormant'. Each row is clickable to the customer drawer and
124
- carries recency so a rep can see who's gone quiet. Sorted by YTD revenue desc (inactive last)."""
125
- t = t or P.today()
126
- book = _book(name)
127
- yf, yt = P.ytd(t)
128
- lf, lt = P.ytd_last_year(t)
129
- this = cust_mod._cust_rev(yf, yt, team_id, book)
130
- last = cust_mod._cust_rev(lf, lt, team_id, book)
131
- lastord = cust_mod._last_order_dates(None, None, team_id, book) # all-time last order = recency
132
- ids = list(book) if book is not None else list(set(this) | set(last))
133
- attrs = cust_mod._partner_attrs(set(ids))
134
- namemap = {r['id']: r.get('name') for r in O.search_read('res.partner', [('id', 'in', ids)], ['name'])}
135
- rows = []
136
- for p in ids:
137
- tr = this.get(p, {}).get('rev', 0.0)
138
- lr = last.get(p, {}).get('rev', 0.0)
139
- a = attrs.get(p, {})
140
- lo = lastord.get(p, '')
141
- recency = (t - dt.date.fromisoformat(lo)).days if lo else None
142
- status = 'Active' if tr > 0 else ('Dormant' if (lr > 0 or lo) else 'Inactive')
143
- rows.append({'pid': p, 'customer': namemap.get(p) or (this.get(p) or last.get(p) or {}).get('name', '?'),
144
- 'rev_ytd': tr, 'rev_ly': lr, 'change': tr - lr, 'yoy_pct': P.yoy_pct(tr, lr),
145
- 'orders': this.get(p, {}).get('orders', 0), 'last_order': lo, 'recency_days': recency,
146
- 'status': status, 'city': a.get('city', '(none)'), 'state': a.get('state', '(none)'),
147
- 'agent': a.get('agent', name)})
148
- rows.sort(key=lambda r: (r['rev_ytd'] <= 0, -r['rev_ytd'], -(r['rev_ly'])))
149
- return rows
150
-
151
-
152
- # ------------------------------------------------------------ top SKUs (with profit/order)
153
- def top_skus(name, t=None, team_id=None, top=30):
154
- """The book's top SKUs YTD (line-level), each with revenue, units, margin and profit/order."""
155
- t = t or P.today()
156
- book = _book(name)
157
- if book is not None and not book:
158
- return []
159
- yf, yt = P.ytd(t)
160
- lex = [('order_partner_id', 'in', list(book))] if book is not None else None
161
- return sales_mod.decompose(yf, yt, team_id, line_extra=lex, top=top)['skus']
162
-
163
-
164
- # ------------------------------------------------------------ returns (book-scoped, consolidated)
165
- def returns_trend(name, n=13, t=None):
166
- return sales_mod.returns_monthly(n, t, partner_ids=_book(name))
167
-
168
-
169
- def returns_headline(name, t=None):
170
- return sales_mod.returns_headline(t, partner_ids=_book(name))
171
-
172
-
173
- # ------------------------------------------------------------ all-agents rollup (the page table)
174
- def _returns_by_agent(t=None):
175
- """{agent_name: returns$ YTD} — credit notes mapped to each customer's agent."""
176
- by_p = sales_mod.returns_by_partner(t)
177
- attrs = cust_mod._partner_attrs(list(by_p))
178
- agg = {}
179
- for pid, amt in by_p.items():
180
- a = (attrs.get(pid) or {}).get('agent') or '(none)'
181
- agg[a] = agg.get(a, 0.0) + amt
182
- return agg
183
-
184
-
185
- def rollup(t=None, team_id=None):
186
- """Every agent ranked by YTD book revenue (+ YoY, customers, orders) with returns $ and return
187
- rate. Reuses the Customers MECE agent rollup, so Σ(agents) == total YTD revenue."""
188
- rows = cust_mod.by_dimension('agent', t, team_id=team_id)
189
- ret = _returns_by_agent(t)
190
- for r in rows:
191
- r['agent'] = r['group']
192
- r['returns'] = ret.get(r['group'], 0.0)
193
- r['return_rate_pct'] = (r['returns'] / r['revenue'] * 100.0) if r.get('revenue') else 0.0
194
- return rows
195
-
196
-
197
- # ------------------------------------------------------------ VALIDATION
198
- def validate(t=None, team_id=None):
199
- """Reconcile the agent rollup to Odoo. (1) Σ(agent book revenue) == total YTD revenue — the
200
- rollup is MECE over customers. (2) A sampled agent's scorecard YTD == its headline YTD."""
201
- t = t or P.today()
202
- yf, yt = P.ytd(t)
203
- checks = []
204
- rows = rollup(t, team_id=team_id)
205
- agent_sum = sum(r['revenue'] for r in rows)
206
- total = O.sum_field('sale.order', sales_mod.order_domain(yf, yt, team_id), 'amount_untaxed')
207
- checks.append({'check': 'YTD revenue: Σ(agent book) == total', 'a': round(agent_sum, 2),
208
- 'b': round(total, 2), 'gap': round(agent_sum - total, 2),
209
- 'ok': abs(agent_sum - total) <= max(1.0, 0.001 * (total or 1))})
210
- # sampled agent: scorecard YTD == headline YTD for the same window
211
- sample = next((r['agent'] for r in rows if r['agent'] not in ('(none)',)), None)
212
- if sample:
213
- sc_ytd = next((s['revenue'] for s in scorecard(sample, t, team_id) if s['key'] == 'ytd'), 0.0)
214
- hl = headline(sample, yf, yt, team_id)['revenue']
215
- checks.append({'check': f'Agent "{sample}": scorecard YTD == headline YTD', 'a': round(sc_ytd, 2),
216
- 'b': round(hl, 2), 'gap': round(sc_ytd - hl, 2), 'ok': abs(sc_ytd - hl) <= 1.0})
217
- checks.extend(validate_measures(t=t, team_id=team_id))
218
- return checks
219
-
220
-
221
- def validate_measures(t=None, team_id=None, days=90):
222
- """⭐⭐ W37-T12 — the minted PER-AGENT lookback columns, against a DIRECT Odoo aggregate.
223
-
224
- ⛔ WHICH AGENT SOURCE, AND THE TICKET REQUIRES IT SAID OUT LOUD: this reconciles ROUTE 1, the
225
- CUSTOMER-MASTER BOOK (`res.partner.agent_ids` -> the mirror's `res_partner.agent_id`, which is
226
- `sales_lines`' `agent` dim). It is NOT the OCA commission route below — measured 2026-08-19,
227
- 9 agents carry route-1 revenue against 12 on commission lines and 17 carrying the flag, so the
228
- two produce materially different rankings and a check that mixed them would be comparing two
229
- different questions and calling the gap an error.
230
-
231
- ⚠ THE ORACLE IS THE ORDER HEADER, not the mirror the columns are served from. `sale.order`
232
- grouped by `partner_id`, mapped to each customer's agent CLIENT-SIDE — a different model, a
233
- different grain and a different code path from `store_query`'s line-level sum, so agreement
234
- between them is evidence rather than tautology.
235
- ⚠ Windowed to the MIRROR'S newest order, like `product_data.validate_measures`, so the
236
- residual is about EDITS to a shared period and not about orders the mirror has never seen.
237
- """
238
- from harness import datastore as DS
239
- from harness import semantic as sem
240
-
241
- t = t or P.today()
242
- checks = []
243
- try:
244
- if not DS.ready():
245
- return [{'check': 'agent lookback measures reconcile to Odoo', 'a': 'no mirror',
246
- 'b': '-', 'ok': False,
247
- 'detail': 'the tenant store is not readable, so this is UNPROVEN, which standing rule '
248
- '8 does not accept as green'}]
249
- except Exception as e: # noqa: BLE001
250
- return [{'check': 'agent lookback measures reconcile to Odoo', 'a': type(e).__name__,
251
- 'b': '-', 'ok': False, 'detail': str(e)[:200]}]
252
-
253
- offer = sem.entity_measures('odoo_agents')
254
- checks.append({'check': 'the agent measure OFFER is non-empty and every key resolves '
255
- '(owner item 4 / R1)',
256
- 'a': len(offer), 'b': '>0', 'ok': bool(offer),
257
- 'detail': {'keys': [m['key'] for m in offer],
258
- 'refused': sem.entity_measure_refusals('odoo_agents')}})
259
- if not offer:
260
- return checks
261
-
262
- con = DS.ro_cursor()
263
- try:
264
- newest = con.execute('SELECT max(date_order) FROM sale_order').fetchone()
265
- finally:
266
- con.close()
267
- d_to = t - dt.timedelta(days=2)
268
- if newest and newest[0]:
269
- try:
270
- d_to = min(d_to, dt.date.fromisoformat(str(newest[0])[:10]) - dt.timedelta(days=1))
271
- except ValueError:
272
- pass
273
- d_from = d_to - dt.timedelta(days=days)
274
- DF, DT = d_from.isoformat(), d_to.isoformat()
275
-
276
- ours = sem.entity_measure_values('odoo_agents', ['revenue'], date_from=DF, date_to=DT,
277
- team_id=team_id, offer=offer)
278
- # THE ORACLE — order headers, grouped by customer, mapped to that customer's agent here.
279
- o = O.get_odoo()
280
- grp = o.read_group('sale.order', sales_mod.order_domain(DF, DT, team_id),
281
- ['partner_id', 'amount_untaxed:sum'], ['partner_id'], lazy=False)
282
- pids = sorted({r['partner_id'][0] for r in grp if r.get('partner_id')})
283
- agent_of = {}
284
- for i in range(0, len(pids), 500):
285
- for p in o.search_read('res.partner', [('id', 'in', pids[i:i + 500])],
286
- ['id', 'agent_ids']):
287
- ag = (p.get('agent_ids') or [])
288
- if ag:
289
- agent_of[p['id']] = ag[0] # the Customers-module convention: agent_ids[0]
290
- theirs = {}
291
- for r in grp:
292
- if not r.get('partner_id'):
293
- continue
294
- a = agent_of.get(r['partner_id'][0])
295
- if a is not None:
296
- theirs[a] = theirs.get(a, 0.0) + r['amount_untaxed']
297
-
298
- ours_tot = round(sum(c.get('revenue', 0) for c in ours.values()), 2)
299
- theirs_tot = round(sum(theirs.values()), 2)
300
- # ⚠ ORDER-HEADER vs LINE-SUM is a REAL basis difference (an order's untaxed total includes
301
- # lines this topic's service filter drops), so the tolerance is a stated 3% rather than a
302
- # cent — and the FIGURE is reported so a drift is readable instead of absorbed.
303
- gap = ours_tot - theirs_tot
304
- checks.append({
305
- 'check': 'per-agent revenue (BOOK route) vs an INDEPENDENT Odoo order-header aggregate '
306
- 'mapped through res.partner.agent_ids',
307
- 'a': ours_tot, 'b': theirs_tot, 'gap': round(gap, 2),
308
- 'ok': bool(theirs_tot) and abs(gap) <= 0.03 * theirs_tot,
309
- 'detail': {'window': [DF, DT], 'agents_ours': len(ours), 'agents_theirs': len(theirs),
310
- 'gap_pct': round(gap / theirs_tot * 100, 3) if theirs_tot else None,
311
- 'route': 'customer-master book (res.partner.agent_ids), NOT the OCA '
312
- 'commission table; the two name different people'}})
313
- # ⛔ AND PER AGENT, because a total can agree while every row is keyed wrong — the join-key
314
- # trap contract C1 names. Here the key is the `res.partner` id at both ends.
315
- off_by = sorted(((abs(ours.get(a, {}).get('revenue', 0.0) - v), a)
316
- for a, v in theirs.items() if v), reverse=True)
317
- bad = [(a, round(ours.get(a, {}).get('revenue', 0.0), 2), round(theirs[a], 2))
318
- for d, a in off_by if d > 0.03 * theirs[a]]
319
- checks.append({
320
- 'check': 'each agent\'s own figure ties (the C1 join-key test: a wrong key agrees in '
321
- 'total and disagrees on every row)',
322
- 'a': len(bad), 'b': 0, 'ok': not bad,
323
- 'detail': {'worst': [(a, ours_v, th_v) for a, ours_v, th_v in bad[:5]],
324
- 'agents_compared': len(theirs)}})
325
- return checks
326
-
327
-
328
- # ------------------------------------------------------------ INVOICE-LINE ATTRIBUTION (2026-07-28)
329
- # A SECOND agent source. Everything above this line attributes by BOOK — the customer's assigned
330
- # agent (res.partner.agent_ids) — over confirmed SALES ORDERS. This section attributes per INVOICE
331
- # LINE, from the OCA sale-commission module, via the semantic layer (topics invoice_lines /
332
- # commission_lines). The two disagree on purpose and answer different questions:
333
- #
334
- # book -> "whose customer is this / who owns the relationship" (order basis)
335
- # invoice -> "what was actually credited to an agent on the billing" + the ONLY source that can
336
- # say what is NOT allocated to an agent (invoice basis)
337
- #
338
- # ⚠ A NAME ON A COMMISSION LINE IS NOT NECESSARILY AN AGENT — `res.partner.agent` is the flag.
339
- # "Anna" and "Shantal Erlich" are internal SALESPEOPLE who carry commission lines; the `agent`
340
- # dim excludes them and `include_salespeople` folds them back in as a clearly-labelled variant.
341
- # See [[invoice-line-agent-commission]].
342
-
343
- _ALLOC_LABEL = {'agent': 'Allocated to an agent', 'salesperson': 'Salesperson only',
344
- 'none': 'Not allocated'}
345
-
346
-
347
- def invoice_line_rollup(t=None, team_id=None, include_salespeople=False):
348
- """Per-name invoice-line revenue + the MECE allocation split, for a YTD window.
349
-
350
- Returns {'by_agent': [...], 'allocation': [...], 'total': float, 'allocated': float,
351
- 'unallocated': float, 'basis': str} — or {'error': msg} when the tenant store is not
352
- ready (this path is store-only; there is no live fallback that stays honest about the
353
- unallocated bucket).
354
- """
355
- import harness.semantic as S
356
- t = t or P.today()
357
- yf, yt = P.ytd(t)
358
- dim = 'commission_name' if include_salespeople else 'agent'
359
- try:
360
- by = S.store_query('invoice_lines', ['invoiced_line_sales'], group_by=[dim],
361
- date_from=yf, date_to=yt, team_id=team_id, limit=200).get('rows') or []
362
- alloc = S.store_query('invoice_lines', ['invoiced_line_sales'], group_by=['allocation'],
363
- date_from=yf, date_to=yt, team_id=team_id, limit=10).get('rows') or []
364
- tot = (S.store_query('invoice_lines', ['invoiced_line_sales'], date_from=yf, date_to=yt,
365
- team_id=team_id).get('rows') or [{}])[0].get('invoiced_line_sales') or 0.0
366
- except Exception as e: # store not ready / model error — say so, don't fake
367
- return {'error': str(e)}
368
- # store_query row shape: the DIM KEY carries the display NAME and `<dim>_id` the raw value
369
- # (allocation -> 'Salesperson only', allocation_id -> 'salesperson'). Reading `<dim>_name`
370
- # returns None for every row and silently renders the whole table as "no agent".
371
- rows = [{'agent': (r.get(dim) or '(no agent on the line)'),
372
- 'revenue': r.get('invoiced_line_sales') or 0.0} for r in by]
373
- rows.sort(key=lambda r: -r['revenue'])
374
- amap = {r.get('allocation_id') or 'none': (r.get('invoiced_line_sales') or 0.0) for r in alloc}
375
- allocated = amap.get('agent', 0.0)
376
- return {
377
- 'by_agent': rows,
378
- 'allocation': [{'bucket': _ALLOC_LABEL[k], 'revenue': amap.get(k, 0.0)}
379
- for k in ('agent', 'salesperson', 'none') if k in amap or True],
380
- 'total': tot, 'allocated': allocated, 'unallocated': tot - allocated,
381
- 'strict_none': amap.get('none', 0.0), 'salesperson_only': amap.get('salesperson', 0.0),
382
- 'window': (yf, yt),
383
- 'basis': ('invoice line · commission names incl. salespeople' if include_salespeople
384
- else 'invoice line · real agents only'),
385
- }
 
1
+ """Agent module — per-agent (res.partner.agent_ids) analytics.
2
+
3
+ An *agent* owns a **book** of customers (the same attribute the Customers module slices by). This
4
+ module reports that book the way Sales/Customers/SKU report the whole company: a period scorecard
5
+ with custom date windows (Today / WTD / Last week / MTD / QTD / YTD / any custom range), a sales
6
+ trend, returns, top SKUs (with profit/order) and the FULL customer list — INCLUDING inactive
7
+ accounts (no recent orders) so a rep sees who they've stopped selling to.
8
+
9
+ Scope: reuses the Sales `order_domain` (Fisch+Royal, excluded accounts removed, state sale/done)
10
+ so numbers tie to every other module. Returns are consolidated (credit notes aren't BU-tagged);
11
+ everything else is BU-filterable via team_id.
12
+ """
13
+ import sys
14
+ import datetime as dt
15
+ from pathlib import Path
16
+ sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
17
+ import core.odoo as O
18
+ import core.periods as P
19
+ import modules.sales as sales_mod
20
+ import modules.customers as cust_mod
21
+
22
+
23
+ def options(t=None, team_id=None):
24
+ """Agent names with book activity — for the page/drawer picker."""
25
+ return cust_mod.agent_options(t, team_id)
26
+
27
+
28
+ def _book(name):
29
+ """frozenset of every partner id assigned to the agent (incl. inactive). None only for 'All'."""
30
+ return cust_mod.agent_partner_ids(name)
31
+
32
+
33
+ def _rev(date_from, date_to, team_id, book):
34
+ return O.sum_field('sale.order', sales_mod.order_domain(date_from, date_to, team_id, partner_ids=book),
35
+ 'amount_untaxed')
36
+
37
+
38
+ def _orders(date_from, date_to, team_id, book):
39
+ return O.get_odoo().search_count('sale.order', sales_mod.order_domain(date_from, date_to, team_id, partner_ids=book))
40
+
41
+
42
+ def _custs(date_from, date_to, team_id, book):
43
+ return O.distinct_count('sale.order', sales_mod.order_domain(date_from, date_to, team_id, partner_ids=book),
44
+ 'partner_id')
45
+
46
+
47
+ # ------------------------------------------------------------ period scorecard (custom dating)
48
+ # Today / WTD / Last week / MTD / QTD / YTD — each carries its window so the UI can decompose it
49
+ # and so "what did this agent sell this/last week" is a click, not a date-math exercise.
50
+ _PERIODS = [('Today', 'today'), ('Week to date', 'wtd'), ('Last week', 'lwk'),
51
+ ('Month to date', 'mtd'), ('Quarter to date', 'qtd'), ('Year to date', 'ytd')]
52
+
53
+
54
+ def _window(key, t):
55
+ if key == 'today':
56
+ return P._d(t), P._d(t)
57
+ if key == 'lwk': # the full prior Mon–Sun week
58
+ f, _tt = P.wtd(t)
59
+ start = dt.date.fromisoformat(f) - dt.timedelta(days=7)
60
+ return start.isoformat(), (dt.date.fromisoformat(f) - dt.timedelta(days=1)).isoformat()
61
+ return {'wtd': P.wtd, 'mtd': P.mtd, 'qtd': P.qtd, 'ytd': P.ytd}[key](t)
62
+
63
+
64
+ def scorecard(name, t=None, team_id=None):
65
+ """Book revenue (+ YoY same-period), orders for Today / WTD / Last week / MTD / QTD / YTD."""
66
+ t = t or P.today()
67
+ book = _book(name)
68
+ out = []
69
+ for label, key in _PERIODS:
70
+ f, tt = _window(key, t)
71
+ wk = key in ('today', 'wtd', 'lwk') # weekday-align the short windows' LY compare
72
+ cf, ct = P.shift_year(f, tt, weeks=wk)
73
+ rev, rev_ly = _rev(f, tt, team_id, book), _rev(cf, ct, team_id, book)
74
+ out.append({'key': key, 'label': label, 'date_from': f, 'date_to': tt, 'cmp_from': cf, 'cmp_to': ct,
75
+ 'revenue': rev, 'revenue_ly': rev_ly, 'yoy_pct': P.yoy_pct(rev, rev_ly),
76
+ 'orders': _orders(f, tt, team_id, book)})
77
+ return out
78
+
79
+
80
+ def headline(name, date_from, date_to, team_id=None):
81
+ """Book KPIs for an ARBITRARY window (custom dating): revenue + YoY (same window LY), orders,
82
+ active customers, AOV and returns $ / return rate."""
83
+ book = _book(name)
84
+ cf, ct = P.shift_year(date_from, date_to, weeks=False)
85
+ rev, rev_ly = _rev(date_from, date_to, team_id, book), _rev(cf, ct, team_id, book)
86
+ orders = _orders(date_from, date_to, team_id, book)
87
+ ret = sales_mod._returns_amt(date_from, date_to, book)
88
+ return {'date_from': date_from, 'date_to': date_to, 'cmp_from': cf, 'cmp_to': ct,
89
+ 'revenue': rev, 'revenue_ly': rev_ly,
90
+ 'yoy_pct': P.yoy_pct(rev, rev_ly), 'orders': orders,
91
+ 'customers': _custs(date_from, date_to, team_id, book), 'aov': (rev / orders) if orders else 0.0,
92
+ 'returns': ret, 'return_rate_pct': (ret / rev * 100.0) if rev else 0.0}
93
+
94
+
95
+ # ------------------------------------------------------------ sales trend
96
+ def _book_monthly_rev(book, date_from, date_to, team_id=None):
97
+ g = O.read_group('sale.order', sales_mod.order_domain(date_from, date_to, team_id, partner_ids=book),
98
+ ['amount_untaxed:sum'], ['date_order:month'], lazy=False)
99
+ out = {}
100
+ for r in g:
101
+ ym = ((r.get('__range') or {}).get('date_order:month') or {}).get('from', '')[:7]
102
+ if ym:
103
+ out[ym] = r.get('amount_untaxed') or 0.0
104
+ return out
105
+
106
+
107
+ def monthly(name, n=13, t=None, team_id=None):
108
+ """Book sales per month vs the same month last year (one month-grouped query over 2 years)."""
109
+ t = t or P.today()
110
+ book = _book(name)
111
+ mrev = _book_monthly_rev(book, dt.date(t.year - 2, t.month, 1).isoformat(), t.isoformat(), team_id)
112
+ rows = []
113
+ for ym, _s, _e in P.month_starts(n, t):
114
+ y, m = int(ym[:4]) - 1, int(ym[5:7])
115
+ this, last = mrev.get(ym, 0.0), mrev.get(f'{y:04d}-{m:02d}', 0.0)
116
+ rows.append({'month': ym, 'revenue': this, 'revenue_ly': last, 'yoy_pct': P.yoy_pct(this, last)})
117
+ return rows
118
+
119
+
120
+ # ------------------------------------------------------------ full customer book (incl. inactive)
121
+ def customers(name, t=None, team_id=None):
122
+ """EVERY customer in the agent's book, including inactive accounts (no YTD/LY orders) — those
123
+ show $0 with status 'Inactive'/'Dormant'. Each row is clickable to the customer drawer and
124
+ carries recency so a rep can see who's gone quiet. Sorted by YTD revenue desc (inactive last)."""
125
+ t = t or P.today()
126
+ book = _book(name)
127
+ yf, yt = P.ytd(t)
128
+ lf, lt = P.ytd_last_year(t)
129
+ this = cust_mod._cust_rev(yf, yt, team_id, book)
130
+ last = cust_mod._cust_rev(lf, lt, team_id, book)
131
+ lastord = cust_mod._last_order_dates(None, None, team_id, book) # all-time last order = recency
132
+ ids = list(book) if book is not None else list(set(this) | set(last))
133
+ attrs = cust_mod._partner_attrs(set(ids))
134
+ namemap = {r['id']: r.get('name') for r in O.search_read('res.partner', [('id', 'in', ids)], ['name'])}
135
+ rows = []
136
+ for p in ids:
137
+ tr = this.get(p, {}).get('rev', 0.0)
138
+ lr = last.get(p, {}).get('rev', 0.0)
139
+ a = attrs.get(p, {})
140
+ lo = lastord.get(p, '')
141
+ recency = (t - dt.date.fromisoformat(lo)).days if lo else None
142
+ status = 'Active' if tr > 0 else ('Dormant' if (lr > 0 or lo) else 'Inactive')
143
+ rows.append({'pid': p, 'customer': namemap.get(p) or (this.get(p) or last.get(p) or {}).get('name', '?'),
144
+ 'rev_ytd': tr, 'rev_ly': lr, 'change': tr - lr, 'yoy_pct': P.yoy_pct(tr, lr),
145
+ 'orders': this.get(p, {}).get('orders', 0), 'last_order': lo, 'recency_days': recency,
146
+ 'status': status, 'city': a.get('city', '(none)'), 'state': a.get('state', '(none)'),
147
+ 'agent': a.get('agent', name)})
148
+ rows.sort(key=lambda r: (r['rev_ytd'] <= 0, -r['rev_ytd'], -(r['rev_ly'])))
149
+ return rows
150
+
151
+
152
+ # ------------------------------------------------------------ top SKUs (with profit/order)
153
+ def top_skus(name, t=None, team_id=None, top=30):
154
+ """The book's top SKUs YTD (line-level), each with revenue, units, margin and profit/order."""
155
+ t = t or P.today()
156
+ book = _book(name)
157
+ if book is not None and not book:
158
+ return []
159
+ yf, yt = P.ytd(t)
160
+ lex = [('order_partner_id', 'in', list(book))] if book is not None else None
161
+ return sales_mod.decompose(yf, yt, team_id, line_extra=lex, top=top)['skus']
162
+
163
+
164
+ # ------------------------------------------------------------ returns (book-scoped, consolidated)
165
+ def returns_trend(name, n=13, t=None):
166
+ return sales_mod.returns_monthly(n, t, partner_ids=_book(name))
167
+
168
+
169
+ def returns_headline(name, t=None):
170
+ return sales_mod.returns_headline(t, partner_ids=_book(name))
171
+
172
+
173
+ # ------------------------------------------------------------ all-agents rollup (the page table)
174
+ def _returns_by_agent(t=None):
175
+ """{agent_name: returns$ YTD} — credit notes mapped to each customer's agent."""
176
+ by_p = sales_mod.returns_by_partner(t)
177
+ attrs = cust_mod._partner_attrs(list(by_p))
178
+ agg = {}
179
+ for pid, amt in by_p.items():
180
+ a = (attrs.get(pid) or {}).get('agent') or '(none)'
181
+ agg[a] = agg.get(a, 0.0) + amt
182
+ return agg
183
+
184
+
185
+ def rollup(t=None, team_id=None):
186
+ """Every agent ranked by YTD book revenue (+ YoY, customers, orders) with returns $ and return
187
+ rate. Reuses the Customers MECE agent rollup, so Σ(agents) == total YTD revenue."""
188
+ rows = cust_mod.by_dimension('agent', t, team_id=team_id)
189
+ ret = _returns_by_agent(t)
190
+ for r in rows:
191
+ r['agent'] = r['group']
192
+ r['returns'] = ret.get(r['group'], 0.0)
193
+ r['return_rate_pct'] = (r['returns'] / r['revenue'] * 100.0) if r.get('revenue') else 0.0
194
+ return rows
195
+
196
+
197
+ # ------------------------------------------------------------ VALIDATION
198
+ def validate(t=None, team_id=None):
199
+ """Reconcile the agent rollup to Odoo. (1) Σ(agent book revenue) == total YTD revenue — the
200
+ rollup is MECE over customers. (2) A sampled agent's scorecard YTD == its headline YTD."""
201
+ t = t or P.today()
202
+ yf, yt = P.ytd(t)
203
+ checks = []
204
+ rows = rollup(t, team_id=team_id)
205
+ agent_sum = sum(r['revenue'] for r in rows)
206
+ total = O.sum_field('sale.order', sales_mod.order_domain(yf, yt, team_id), 'amount_untaxed')
207
+ checks.append({'check': 'YTD revenue: Σ(agent book) == total', 'a': round(agent_sum, 2),
208
+ 'b': round(total, 2), 'gap': round(agent_sum - total, 2),
209
+ 'ok': abs(agent_sum - total) <= max(1.0, 0.001 * (total or 1))})
210
+ # sampled agent: scorecard YTD == headline YTD for the same window
211
+ sample = next((r['agent'] for r in rows if r['agent'] not in ('(none)',)), None)
212
+ if sample:
213
+ sc_ytd = next((s['revenue'] for s in scorecard(sample, t, team_id) if s['key'] == 'ytd'), 0.0)
214
+ hl = headline(sample, yf, yt, team_id)['revenue']
215
+ checks.append({'check': f'Agent "{sample}": scorecard YTD == headline YTD', 'a': round(sc_ytd, 2),
216
+ 'b': round(hl, 2), 'gap': round(sc_ytd - hl, 2), 'ok': abs(sc_ytd - hl) <= 1.0})
217
+ checks.extend(validate_measures(t=t, team_id=team_id))
218
+ return checks
219
+
220
+
221
+ def validate_measures(t=None, team_id=None, days=90):
222
+ """⭐⭐ W37-T12 — the minted PER-AGENT lookback columns, against a DIRECT Odoo aggregate.
223
+
224
+ ⛔ WHICH AGENT SOURCE, AND THE TICKET REQUIRES IT SAID OUT LOUD: this reconciles ROUTE 1, the
225
+ CUSTOMER-MASTER BOOK (`res.partner.agent_ids` -> the mirror's `res_partner.agent_id`, which is
226
+ `sales_lines`' `agent` dim). It is NOT the OCA commission route below — measured 2026-08-19,
227
+ 9 agents carry route-1 revenue against 12 on commission lines and 17 carrying the flag, so the
228
+ two produce materially different rankings and a check that mixed them would be comparing two
229
+ different questions and calling the gap an error.
230
+
231
+ ⚠ THE ORACLE IS THE ORDER HEADER, not the mirror the columns are served from. `sale.order`
232
+ grouped by `partner_id`, mapped to each customer's agent CLIENT-SIDE — a different model, a
233
+ different grain and a different code path from `store_query`'s line-level sum, so agreement
234
+ between them is evidence rather than tautology.
235
+ ⚠ Windowed to the MIRROR'S newest order, like `product_data.validate_measures`, so the
236
+ residual is about EDITS to a shared period and not about orders the mirror has never seen.
237
+ """
238
+ from harness import datastore as DS
239
+ from harness import semantic as sem
240
+
241
+ t = t or P.today()
242
+ checks = []
243
+ try:
244
+ if not DS.ready():
245
+ return [{'check': 'agent lookback measures reconcile to Odoo', 'a': 'no mirror',
246
+ 'b': '-', 'ok': False,
247
+ 'detail': 'the tenant store is not readable, so this is UNPROVEN, which standing rule '
248
+ '8 does not accept as green'}]
249
+ except Exception as e: # noqa: BLE001
250
+ return [{'check': 'agent lookback measures reconcile to Odoo', 'a': type(e).__name__,
251
+ 'b': '-', 'ok': False, 'detail': str(e)[:200]}]
252
+
253
+ offer = sem.entity_measures('odoo_agents')
254
+ checks.append({'check': 'the agent measure OFFER is non-empty and every key resolves '
255
+ '(owner item 4 / R1)',
256
+ 'a': len(offer), 'b': '>0', 'ok': bool(offer),
257
+ 'detail': {'keys': [m['key'] for m in offer],
258
+ 'refused': sem.entity_measure_refusals('odoo_agents')}})
259
+ if not offer:
260
+ return checks
261
+
262
+ con = DS.ro_cursor()
263
+ try:
264
+ newest = con.execute('SELECT max(date_order) FROM sale_order').fetchone()
265
+ finally:
266
+ con.close()
267
+ d_to = t - dt.timedelta(days=2)
268
+ if newest and newest[0]:
269
+ try:
270
+ d_to = min(d_to, dt.date.fromisoformat(str(newest[0])[:10]) - dt.timedelta(days=1))
271
+ except ValueError:
272
+ pass
273
+ d_from = d_to - dt.timedelta(days=days)
274
+ DF, DT = d_from.isoformat(), d_to.isoformat()
275
+
276
+ ours = sem.entity_measure_values('odoo_agents', ['revenue'], date_from=DF, date_to=DT,
277
+ team_id=team_id, offer=offer)
278
+ # THE ORACLE — order headers, grouped by customer, mapped to that customer's agent here.
279
+ o = O.get_odoo()
280
+ grp = o.read_group('sale.order', sales_mod.order_domain(DF, DT, team_id),
281
+ ['partner_id', 'amount_untaxed:sum'], ['partner_id'], lazy=False)
282
+ pids = sorted({r['partner_id'][0] for r in grp if r.get('partner_id')})
283
+ agent_of = {}
284
+ for i in range(0, len(pids), 500):
285
+ for p in o.search_read('res.partner', [('id', 'in', pids[i:i + 500])],
286
+ ['id', 'agent_ids']):
287
+ ag = (p.get('agent_ids') or [])
288
+ if ag:
289
+ agent_of[p['id']] = ag[0] # the Customers-module convention: agent_ids[0]
290
+ theirs = {}
291
+ for r in grp:
292
+ if not r.get('partner_id'):
293
+ continue
294
+ a = agent_of.get(r['partner_id'][0])
295
+ if a is not None:
296
+ theirs[a] = theirs.get(a, 0.0) + r['amount_untaxed']
297
+
298
+ ours_tot = round(sum(c.get('revenue', 0) for c in ours.values()), 2)
299
+ theirs_tot = round(sum(theirs.values()), 2)
300
+ # ⚠ ORDER-HEADER vs LINE-SUM is a REAL basis difference (an order's untaxed total includes
301
+ # lines this topic's service filter drops), so the tolerance is a stated 3% rather than a
302
+ # cent — and the FIGURE is reported so a drift is readable instead of absorbed.
303
+ gap = ours_tot - theirs_tot
304
+ checks.append({
305
+ 'check': 'per-agent revenue (BOOK route) vs an INDEPENDENT Odoo order-header aggregate '
306
+ 'mapped through res.partner.agent_ids',
307
+ 'a': ours_tot, 'b': theirs_tot, 'gap': round(gap, 2),
308
+ 'ok': bool(theirs_tot) and abs(gap) <= 0.03 * theirs_tot,
309
+ 'detail': {'window': [DF, DT], 'agents_ours': len(ours), 'agents_theirs': len(theirs),
310
+ 'gap_pct': round(gap / theirs_tot * 100, 3) if theirs_tot else None,
311
+ 'route': 'customer-master book (res.partner.agent_ids), NOT the OCA '
312
+ 'commission table; the two name different people'}})
313
+ # ⛔ AND PER AGENT, because a total can agree while every row is keyed wrong — the join-key
314
+ # trap contract C1 names. Here the key is the `res.partner` id at both ends.
315
+ off_by = sorted(((abs(ours.get(a, {}).get('revenue', 0.0) - v), a)
316
+ for a, v in theirs.items() if v), reverse=True)
317
+ bad = [(a, round(ours.get(a, {}).get('revenue', 0.0), 2), round(theirs[a], 2))
318
+ for d, a in off_by if d > 0.03 * theirs[a]]
319
+ checks.append({
320
+ 'check': 'each agent\'s own figure ties (the C1 join-key test: a wrong key agrees in '
321
+ 'total and disagrees on every row)',
322
+ 'a': len(bad), 'b': 0, 'ok': not bad,
323
+ 'detail': {'worst': [(a, ours_v, th_v) for a, ours_v, th_v in bad[:5]],
324
+ 'agents_compared': len(theirs)}})
325
+ return checks
326
+
327
+
328
+ # ------------------------------------------------------------ INVOICE-LINE ATTRIBUTION (2026-07-28)
329
+ # A SECOND agent source. Everything above this line attributes by BOOK — the customer's assigned
330
+ # agent (res.partner.agent_ids) — over confirmed SALES ORDERS. This section attributes per INVOICE
331
+ # LINE, from the OCA sale-commission module, via the semantic layer (topics invoice_lines /
332
+ # commission_lines). The two disagree on purpose and answer different questions:
333
+ #
334
+ # book -> "whose customer is this / who owns the relationship" (order basis)
335
+ # invoice -> "what was actually credited to an agent on the billing" + the ONLY source that can
336
+ # say what is NOT allocated to an agent (invoice basis)
337
+ #
338
+ # ⚠ A NAME ON A COMMISSION LINE IS NOT NECESSARILY AN AGENT — `res.partner.agent` is the flag.
339
+ # "Anna" and "Shantal Erlich" are internal SALESPEOPLE who carry commission lines; the `agent`
340
+ # dim excludes them and `include_salespeople` folds them back in as a clearly-labelled variant.
341
+ # See [[invoice-line-agent-commission]].
342
+
343
+ _ALLOC_LABEL = {'agent': 'Allocated to an agent', 'salesperson': 'Salesperson only',
344
+ 'none': 'Not allocated'}
345
+
346
+
347
+ def invoice_line_rollup(t=None, team_id=None, include_salespeople=False):
348
+ """Per-name invoice-line revenue + the MECE allocation split, for a YTD window.
349
+
350
+ Returns {'by_agent': [...], 'allocation': [...], 'total': float, 'allocated': float,
351
+ 'unallocated': float, 'basis': str} — or {'error': msg} when the tenant store is not
352
+ ready (this path is store-only; there is no live fallback that stays honest about the
353
+ unallocated bucket).
354
+ """
355
+ import harness.semantic as S
356
+ t = t or P.today()
357
+ yf, yt = P.ytd(t)
358
+ dim = 'commission_name' if include_salespeople else 'agent'
359
+ try:
360
+ by = S.store_query('invoice_lines', ['invoiced_line_sales'], group_by=[dim],
361
+ date_from=yf, date_to=yt, team_id=team_id, limit=200).get('rows') or []
362
+ alloc = S.store_query('invoice_lines', ['invoiced_line_sales'], group_by=['allocation'],
363
+ date_from=yf, date_to=yt, team_id=team_id, limit=10).get('rows') or []
364
+ tot = (S.store_query('invoice_lines', ['invoiced_line_sales'], date_from=yf, date_to=yt,
365
+ team_id=team_id).get('rows') or [{}])[0].get('invoiced_line_sales') or 0.0
366
+ except Exception as e: # store not ready / model error — say so, don't fake
367
+ return {'error': str(e)}
368
+ # store_query row shape: the DIM KEY carries the display NAME and `<dim>_id` the raw value
369
+ # (allocation -> 'Salesperson only', allocation_id -> 'salesperson'). Reading `<dim>_name`
370
+ # returns None for every row and silently renders the whole table as "no agent".
371
+ rows = [{'agent': (r.get(dim) or '(no agent on the line)'),
372
+ 'revenue': r.get('invoiced_line_sales') or 0.0} for r in by]
373
+ rows.sort(key=lambda r: -r['revenue'])
374
+ amap = {r.get('allocation_id') or 'none': (r.get('invoiced_line_sales') or 0.0) for r in alloc}
375
+ allocated = amap.get('agent', 0.0)
376
+ return {
377
+ 'by_agent': rows,
378
+ 'allocation': [{'bucket': _ALLOC_LABEL[k], 'revenue': amap.get(k, 0.0)}
379
+ for k in ('agent', 'salesperson', 'none') if k in amap or True],
380
+ 'total': tot, 'allocated': allocated, 'unallocated': tot - allocated,
381
+ 'strict_none': amap.get('none', 0.0), 'salesperson_only': amap.get('salesperson', 0.0),
382
+ 'window': (yf, yt),
383
+ 'basis': ('invoice line · commission names incl. salespeople' if include_salespeople
384
+ else 'invoice line · real agents only'),
385
+ }
platform/modules/collections_send.py CHANGED
@@ -1,267 +1,267 @@
1
- """Collections statements — the send layer behind the Collections page's Statements section.
2
-
3
- Folded in from the standalone collections_app (2026-07-05). THE one sanctioned exception to the
4
- app's read-only-on-Odoo rule, unchanged from the standalone tool: WRITE is whitelisted to exactly
5
- one operation —
6
-
7
- create on mail.mail (queueing an outbound statement email)
8
-
9
- A mail.mail record with state='outgoing' is picked up by Odoo's "Mail: Email Queue Manager" cron
10
- (every ~15 min) and delivered through the company's Office 365 relay. Because we set model/res_id,
11
- each sent statement also appears in the customer's chatter in Odoo — the audit log lives where AR
12
- already works. This module has its OWN narrow XML-RPC client; the app-wide odoo_client stays
13
- hard-blocking on all writes. The UI gates the section to admin users; SAFE_MODE is enforced HERE
14
- in the data layer so the UI cannot bypass it.
15
-
16
- Env (Space secrets / .env): ODOO_URL, ODOO_DB, ODOO_USER, ODOO_API_KEY
17
- Optional: SAFE_MODE (default ON), SAFE_RECIPIENTS, SENDER_NAME, SENDER_EMAIL, REPLY_TO,
18
- COMPANY_NAME, ROYAL_MAIL_SERVER_ID, ROYAL_AUTHOR_ID
19
- """
20
- import os
21
- import datetime as dt
22
- import xmlrpc.client
23
- from pathlib import Path
24
-
25
- try: # self-contained: load credentials from the app root .env if present (HF uses Secrets)
26
- from dotenv import load_dotenv
27
- load_dotenv(Path(__file__).resolve().parents[1] / '.env')
28
- except Exception:
29
- pass
30
-
31
- WRITE_WHITELIST = {('mail.mail', 'create')}
32
-
33
- EXCLUDE_NAMES = {'GIFTWARE DEALS'} # the Amazon channel — not part of Fisch or Royal collections
34
- DOMAIN = [('followup_reminder_type', '=', 'automatic'), ('credit', '>', 1)]
35
-
36
- COMPANY = os.environ.get('COMPANY_NAME', 'Royal Imports')
37
-
38
- # --- Sender identity (statements go out AS Royal Imports) ---
39
- # Odoo routes outbound mail to the matching ir.mail_server by from_filter, and the Office 365
40
- # relay only accepts sends as its authenticated address. The "Office 365 - Royal" server (id 2)
41
- # authenticates as contact@royalimports.com with from_filter='contact@royalimports.com' — so the
42
- # From MUST be that address for delivery to succeed. Friendly display name; replies routed to AR.
43
- SENDER_NAME = os.environ.get('SENDER_NAME', 'Royal Imports Accounts Receivable')
44
- SENDER_EMAIL = os.environ.get('SENDER_EMAIL', 'contact@royalimports.com')
45
- REPLY_TO = os.environ.get('REPLY_TO', 'accounting@royalimports.com')
46
- ROYAL_MAIL_SERVER_ID = int(os.environ.get('ROYAL_MAIL_SERVER_ID', '2'))
47
- ROYAL_AUTHOR_ID = int(os.environ.get('ROYAL_AUTHOR_ID', '8978')) # "Royal Imports" partner
48
-
49
- SENDER_DISPLAY = f'"{SENDER_NAME}" <{SENDER_EMAIL}>'
50
-
51
-
52
- class WriteBlocked(RuntimeError):
53
- pass
54
-
55
-
56
- class SafeModeBlocked(RuntimeError):
57
- pass
58
-
59
-
60
- # --- Testing guardrail -------------------------------------------------------
61
- # While SAFE_MODE is on, NO email can be queued to any address outside the allow-list — enforced
62
- # here in the data layer so the UI cannot bypass it. Default: ON. To go live for real customers,
63
- # set the Space secret SAFE_MODE=0.
64
- SAFE_MODE = os.environ.get('SAFE_MODE', '1').strip().lower() not in ('0', 'false', 'no', '')
65
- SAFE_RECIPIENTS = {a.strip().lower() for a in
66
- os.environ.get('SAFE_RECIPIENTS', 'farhan@teamroyalimports.com').split(',')
67
- if a.strip()}
68
-
69
-
70
- def safe_recipient_ok(addr):
71
- return (not SAFE_MODE) or (str(addr or '').strip().lower() in SAFE_RECIPIENTS)
72
-
73
-
74
- class Odoo:
75
- """Narrow client: read anything, write ONLY the whitelisted mail.mail create."""
76
-
77
- def __init__(self):
78
- self.url = os.environ.get('ODOO_URL', '').rstrip('/')
79
- self.db = os.environ.get('ODOO_DB', '')
80
- self.user = os.environ.get('ODOO_USER', '')
81
- self.key = os.environ.get('ODOO_API_KEY', '')
82
- missing = [k for k, v in [('ODOO_URL', self.url), ('ODOO_DB', self.db),
83
- ('ODOO_USER', self.user), ('ODOO_API_KEY', self.key)] if not v]
84
- if missing:
85
- raise RuntimeError(f"Missing secrets: {', '.join(missing)}")
86
- common = xmlrpc.client.ServerProxy(f'{self.url}/xmlrpc/2/common')
87
- self.uid = common.authenticate(self.db, self.user, self.key, {})
88
- if not self.uid:
89
- raise RuntimeError('Odoo authentication failed')
90
- self.models = xmlrpc.client.ServerProxy(f'{self.url}/xmlrpc/2/object')
91
-
92
- def _exec(self, model, method, args, kwargs=None):
93
- mutating = method in ('write', 'create', 'unlink', 'copy') or \
94
- any(method.startswith(p) for p in ('action_', 'button_', 'do_', 'send_',
95
- 'set_', 'update_', 'process_'))
96
- if mutating and (model, method) not in WRITE_WHITELIST:
97
- raise WriteBlocked(f'{method} on {model} is not allowed from this app')
98
- return self.models.execute_kw(self.db, self.uid, self.key,
99
- model, method, args, kwargs or {})
100
-
101
- def search_read(self, model, domain=None, fields=None, limit=None, order=None):
102
- kw = {'fields': fields or []}
103
- if limit is not None:
104
- kw['limit'] = limit
105
- if order:
106
- kw['order'] = order
107
- return self._exec(model, 'search_read', [domain or []], kw)
108
-
109
- def queue_mail(self, payload):
110
- """The single allowed write: queue an outbound email."""
111
- return self._exec('mail.mail', 'create', [payload])
112
-
113
-
114
- # --------------------------------------------------------------- data builders
115
- def load_collection_list(odoo):
116
- """The saved follow-up filter (Reminders=Automatic, Receivable>$1), GIFTWARE excluded,
117
- with priority tiers."""
118
- partners = odoo.search_read('res.partner', DOMAIN,
119
- ['name', 'credit', 'total_overdue', 'followup_status',
120
- 'followup_next_action_date', 'followup_responsible_id',
121
- 'email', 'phone', 'mobile'])
122
- partners = [p for p in partners if (p['name'] or '').strip().upper() not in EXCLUDE_NAMES]
123
- pids = [p['id'] for p in partners]
124
-
125
- docs = odoo.search_read('account.move',
126
- [('move_type', 'in', ['out_invoice', 'out_refund']), ('state', '=', 'posted'),
127
- ('payment_state', 'in', ['not_paid', 'partial']), ('partner_id', 'in', pids)],
128
- ['name', 'partner_id', 'move_type', 'invoice_date', 'invoice_date_due',
129
- 'amount_total', 'amount_residual_signed'])
130
-
131
- today = dt.date.today()
132
- by_partner = {}
133
- for d in docs:
134
- pid = d['partner_id'][0]
135
- due = d.get('invoice_date_due')
136
- try:
137
- days = (today - dt.date.fromisoformat(due)).days if due else 0
138
- except Exception:
139
- days = 0
140
- d['days_overdue'] = max(days, 0)
141
- d['open_signed'] = d['amount_residual_signed']
142
- by_partner.setdefault(pid, []).append(d)
143
-
144
- rows = []
145
- for p in partners:
146
- pid = p['id']
147
- odoo_overdue = float(p.get('total_overdue') or 0)
148
- docs_p = sorted(by_partner.get(pid, []), key=lambda x: x.get('invoice_date_due') or '')
149
- inv_overdue = sum(d['open_signed'] for d in docs_p if d['days_overdue'] > 0)
150
- oldest = max((d['days_overdue'] for d in docs_p), default=0)
151
- gap = inv_overdue - odoo_overdue
152
- reconcile = abs(gap) > 50
153
-
154
- if odoo_overdue >= 5000 or (odoo_overdue > 0 and oldest > 90):
155
- tier = 'A-Urgent'
156
- elif odoo_overdue >= 1000 or (odoo_overdue > 0 and oldest > 30):
157
- tier = 'B-Active'
158
- elif odoo_overdue > 0:
159
- tier = 'C-Light'
160
- else:
161
- tier = 'Monitor'
162
-
163
- rows.append({
164
- 'partner_id': pid,
165
- 'Customer': p['name'],
166
- 'Tier': tier,
167
- 'Overdue': odoo_overdue,
168
- 'Receivable': float(p.get('credit') or 0),
169
- 'Oldest (days)': oldest,
170
- 'Open Docs': len(docs_p),
171
- 'Email': p.get('email') or '',
172
- 'Phone': p.get('phone') or p.get('mobile') or '',
173
- 'Status': (p.get('followup_status') or '').replace('_', ' '),
174
- 'Reconcile?': 'YES' if reconcile else '',
175
- '_docs': docs_p,
176
- })
177
- tier_rank = {'A-Urgent': 0, 'B-Active': 1, 'C-Light': 2, 'Monitor': 3}
178
- rows.sort(key=lambda r: (tier_rank[r['Tier']], -r['Overdue']))
179
- return rows
180
-
181
-
182
- # --------------------------------------------------------------- statement email
183
- # ⛔ STANDING RULE 2, AND THIS IS THE ONE STRING IN THE PRODUCT THAT LEAVES THE BUILDING.
184
- # Every other finding `web_prose` reports is copy on a screen somebody here opens; this is the
185
- # SUBJECT LINE of mail queued to a real debtor, over the single sanctioned Odoo write. It read
186
- # 'Statement of Account (em dash) {company} (em dash) {month}' until W36-T42.
187
- DEFAULT_SUBJECT = 'Statement of Account from {company}, {month}'
188
- DEFAULT_INTRO = (
189
- 'Dear {customer},<br><br>'
190
- 'Please find below your current statement of account with {company}. '
191
- 'According to our records, the following invoices remain open:'
192
- )
193
- DEFAULT_FOOTER = (
194
- 'If you have already sent payment, please disregard this notice, and thank you. '
195
- 'For any questions about an invoice, simply reply to this email.<br><br>'
196
- 'Thank you for your business,<br>{company}<br>Accounts Receivable'
197
- )
198
-
199
-
200
- def render_statement_html(row, intro_tpl=DEFAULT_INTRO, footer_tpl=DEFAULT_FOOTER):
201
- month = dt.date.today().strftime('%B %Y')
202
- intro = intro_tpl.format(customer=row['Customer'], company=COMPANY, month=month)
203
- footer = footer_tpl.format(customer=row['Customer'], company=COMPANY, month=month)
204
-
205
- lines = []
206
- total_open = 0.0
207
- for d in row['_docs']:
208
- kind = 'Credit Note' if d['move_type'] == 'out_refund' else 'Invoice'
209
- amt = d['open_signed']
210
- total_open += amt
211
- overdue_txt = f"{d['days_overdue']}d overdue" if d['days_overdue'] > 0 else 'current'
212
- color = '#C0392B' if d['days_overdue'] > 0 else '#1F4E78'
213
- lines.append(
214
- f"<tr><td style='padding:6px 10px;border-bottom:1px solid #e3e8ef'>{d['name']} <span style='color:#888'>({kind})</span></td>"
215
- f"<td style='padding:6px 10px;border-bottom:1px solid #e3e8ef'>{d.get('invoice_date') or ''}</td>"
216
- f"<td style='padding:6px 10px;border-bottom:1px solid #e3e8ef'>{d.get('invoice_date_due') or ''}</td>"
217
- f"<td style='padding:6px 10px;border-bottom:1px solid #e3e8ef;color:{color}'>{overdue_txt}</td>"
218
- f"<td style='padding:6px 10px;border-bottom:1px solid #e3e8ef;text-align:right'>${amt:,.2f}</td></tr>")
219
-
220
- table = (
221
- "<table style='border-collapse:collapse;font-size:14px;margin:14px 0'>"
222
- "<tr style='background:#1F4E78;color:#fff'>"
223
- "<th style='padding:7px 10px;text-align:left'>Document</th>"
224
- "<th style='padding:7px 10px;text-align:left'>Date</th>"
225
- "<th style='padding:7px 10px;text-align:left'>Due</th>"
226
- "<th style='padding:7px 10px;text-align:left'>Status</th>"
227
- "<th style='padding:7px 10px;text-align:right'>Open Balance</th></tr>"
228
- + ''.join(lines) +
229
- f"<tr><td colspan='4' style='padding:8px 10px;font-weight:bold;text-align:right'>Total open</td>"
230
- f"<td style='padding:8px 10px;font-weight:bold;text-align:right'>${total_open:,.2f}</td></tr>"
231
- f"<tr><td colspan='4' style='padding:2px 10px;font-weight:bold;text-align:right;color:#C0392B'>Of which overdue</td>"
232
- f"<td style='padding:2px 10px;font-weight:bold;text-align:right;color:#C0392B'>${row['Overdue']:,.2f}</td></tr>"
233
- "</table>")
234
-
235
- return (f"<div style='font-family:Calibri,Arial,sans-serif;color:#1a1a1a;font-size:14px'>"
236
- f"{intro}{table}{footer}</div>")
237
-
238
-
239
- def queue_statement(odoo, row, subject_tpl=DEFAULT_SUBJECT,
240
- intro_tpl=DEFAULT_INTRO, footer_tpl=DEFAULT_FOOTER,
241
- override_to=None):
242
- """Queue one statement email in Odoo. Returns mail.mail id.
243
- override_to: send to a different address (used by the test-send button)."""
244
- to = override_to or row['Email']
245
- if not to:
246
- raise ValueError(f"{row['Customer']} has no email address")
247
- # Hard guardrail — refuse any recipient outside the allow-list while SAFE_MODE is on.
248
- if not safe_recipient_ok(to):
249
- raise SafeModeBlocked(
250
- f"Guardrail ON: refusing to email {to}. Only {', '.join(sorted(SAFE_RECIPIENTS))} "
251
- f"is allowed right now. (Set SAFE_MODE=0 to send to real customers.)")
252
- month = dt.date.today().strftime('%B %Y')
253
- subject = subject_tpl.format(customer=row['Customer'], company=COMPANY, month=month)
254
- payload = {
255
- 'subject': subject,
256
- 'body_html': render_statement_html(row, intro_tpl, footer_tpl),
257
- 'email_to': to,
258
- 'email_from': SENDER_DISPLAY, # From: Royal Imports
259
- 'reply_to': REPLY_TO, # replies -> AR
260
- 'mail_server_id': ROYAL_MAIL_SERVER_ID, # force Royal O365 relay
261
- 'author_id': ROYAL_AUTHOR_ID, # clean attribution in chatter
262
- 'state': 'outgoing',
263
- 'auto_delete': False,
264
- 'model': 'res.partner',
265
- 'res_id': row['partner_id'],
266
- }
267
- return odoo.queue_mail(payload)
 
1
+ """Collections statements — the send layer behind the Collections page's Statements section.
2
+
3
+ Folded in from the standalone collections_app (2026-07-05). THE one sanctioned exception to the
4
+ app's read-only-on-Odoo rule, unchanged from the standalone tool: WRITE is whitelisted to exactly
5
+ one operation —
6
+
7
+ create on mail.mail (queueing an outbound statement email)
8
+
9
+ A mail.mail record with state='outgoing' is picked up by Odoo's "Mail: Email Queue Manager" cron
10
+ (every ~15 min) and delivered through the company's Office 365 relay. Because we set model/res_id,
11
+ each sent statement also appears in the customer's chatter in Odoo — the audit log lives where AR
12
+ already works. This module has its OWN narrow XML-RPC client; the app-wide odoo_client stays
13
+ hard-blocking on all writes. The UI gates the section to admin users; SAFE_MODE is enforced HERE
14
+ in the data layer so the UI cannot bypass it.
15
+
16
+ Env (Space secrets / .env): ODOO_URL, ODOO_DB, ODOO_USER, ODOO_API_KEY
17
+ Optional: SAFE_MODE (default ON), SAFE_RECIPIENTS, SENDER_NAME, SENDER_EMAIL, REPLY_TO,
18
+ COMPANY_NAME, ROYAL_MAIL_SERVER_ID, ROYAL_AUTHOR_ID
19
+ """
20
+ import os
21
+ import datetime as dt
22
+ import xmlrpc.client
23
+ from pathlib import Path
24
+
25
+ try: # self-contained: load credentials from the app root .env if present (HF uses Secrets)
26
+ from dotenv import load_dotenv
27
+ load_dotenv(Path(__file__).resolve().parents[1] / '.env')
28
+ except Exception:
29
+ pass
30
+
31
+ WRITE_WHITELIST = {('mail.mail', 'create')}
32
+
33
+ EXCLUDE_NAMES = {'GIFTWARE DEALS'} # the Amazon channel — not part of Fisch or Royal collections
34
+ DOMAIN = [('followup_reminder_type', '=', 'automatic'), ('credit', '>', 1)]
35
+
36
+ COMPANY = os.environ.get('COMPANY_NAME', 'Royal Imports')
37
+
38
+ # --- Sender identity (statements go out AS Royal Imports) ---
39
+ # Odoo routes outbound mail to the matching ir.mail_server by from_filter, and the Office 365
40
+ # relay only accepts sends as its authenticated address. The "Office 365 - Royal" server (id 2)
41
+ # authenticates as contact@royalimports.com with from_filter='contact@royalimports.com' — so the
42
+ # From MUST be that address for delivery to succeed. Friendly display name; replies routed to AR.
43
+ SENDER_NAME = os.environ.get('SENDER_NAME', 'Royal Imports Accounts Receivable')
44
+ SENDER_EMAIL = os.environ.get('SENDER_EMAIL', 'contact@royalimports.com')
45
+ REPLY_TO = os.environ.get('REPLY_TO', 'accounting@royalimports.com')
46
+ ROYAL_MAIL_SERVER_ID = int(os.environ.get('ROYAL_MAIL_SERVER_ID', '2'))
47
+ ROYAL_AUTHOR_ID = int(os.environ.get('ROYAL_AUTHOR_ID', '8978')) # "Royal Imports" partner
48
+
49
+ SENDER_DISPLAY = f'"{SENDER_NAME}" <{SENDER_EMAIL}>'
50
+
51
+
52
+ class WriteBlocked(RuntimeError):
53
+ pass
54
+
55
+
56
+ class SafeModeBlocked(RuntimeError):
57
+ pass
58
+
59
+
60
+ # --- Testing guardrail -------------------------------------------------------
61
+ # While SAFE_MODE is on, NO email can be queued to any address outside the allow-list — enforced
62
+ # here in the data layer so the UI cannot bypass it. Default: ON. To go live for real customers,
63
+ # set the Space secret SAFE_MODE=0.
64
+ SAFE_MODE = os.environ.get('SAFE_MODE', '1').strip().lower() not in ('0', 'false', 'no', '')
65
+ SAFE_RECIPIENTS = {a.strip().lower() for a in
66
+ os.environ.get('SAFE_RECIPIENTS', 'farhan@teamroyalimports.com').split(',')
67
+ if a.strip()}
68
+
69
+
70
+ def safe_recipient_ok(addr):
71
+ return (not SAFE_MODE) or (str(addr or '').strip().lower() in SAFE_RECIPIENTS)
72
+
73
+
74
+ class Odoo:
75
+ """Narrow client: read anything, write ONLY the whitelisted mail.mail create."""
76
+
77
+ def __init__(self):
78
+ self.url = os.environ.get('ODOO_URL', '').rstrip('/')
79
+ self.db = os.environ.get('ODOO_DB', '')
80
+ self.user = os.environ.get('ODOO_USER', '')
81
+ self.key = os.environ.get('ODOO_API_KEY', '')
82
+ missing = [k for k, v in [('ODOO_URL', self.url), ('ODOO_DB', self.db),
83
+ ('ODOO_USER', self.user), ('ODOO_API_KEY', self.key)] if not v]
84
+ if missing:
85
+ raise RuntimeError(f"Missing secrets: {', '.join(missing)}")
86
+ common = xmlrpc.client.ServerProxy(f'{self.url}/xmlrpc/2/common')
87
+ self.uid = common.authenticate(self.db, self.user, self.key, {})
88
+ if not self.uid:
89
+ raise RuntimeError('Odoo authentication failed')
90
+ self.models = xmlrpc.client.ServerProxy(f'{self.url}/xmlrpc/2/object')
91
+
92
+ def _exec(self, model, method, args, kwargs=None):
93
+ mutating = method in ('write', 'create', 'unlink', 'copy') or \
94
+ any(method.startswith(p) for p in ('action_', 'button_', 'do_', 'send_',
95
+ 'set_', 'update_', 'process_'))
96
+ if mutating and (model, method) not in WRITE_WHITELIST:
97
+ raise WriteBlocked(f'{method} on {model} is not allowed from this app')
98
+ return self.models.execute_kw(self.db, self.uid, self.key,
99
+ model, method, args, kwargs or {})
100
+
101
+ def search_read(self, model, domain=None, fields=None, limit=None, order=None):
102
+ kw = {'fields': fields or []}
103
+ if limit is not None:
104
+ kw['limit'] = limit
105
+ if order:
106
+ kw['order'] = order
107
+ return self._exec(model, 'search_read', [domain or []], kw)
108
+
109
+ def queue_mail(self, payload):
110
+ """The single allowed write: queue an outbound email."""
111
+ return self._exec('mail.mail', 'create', [payload])
112
+
113
+
114
+ # --------------------------------------------------------------- data builders
115
+ def load_collection_list(odoo):
116
+ """The saved follow-up filter (Reminders=Automatic, Receivable>$1), GIFTWARE excluded,
117
+ with priority tiers."""
118
+ partners = odoo.search_read('res.partner', DOMAIN,
119
+ ['name', 'credit', 'total_overdue', 'followup_status',
120
+ 'followup_next_action_date', 'followup_responsible_id',
121
+ 'email', 'phone', 'mobile'])
122
+ partners = [p for p in partners if (p['name'] or '').strip().upper() not in EXCLUDE_NAMES]
123
+ pids = [p['id'] for p in partners]
124
+
125
+ docs = odoo.search_read('account.move',
126
+ [('move_type', 'in', ['out_invoice', 'out_refund']), ('state', '=', 'posted'),
127
+ ('payment_state', 'in', ['not_paid', 'partial']), ('partner_id', 'in', pids)],
128
+ ['name', 'partner_id', 'move_type', 'invoice_date', 'invoice_date_due',
129
+ 'amount_total', 'amount_residual_signed'])
130
+
131
+ today = dt.date.today()
132
+ by_partner = {}
133
+ for d in docs:
134
+ pid = d['partner_id'][0]
135
+ due = d.get('invoice_date_due')
136
+ try:
137
+ days = (today - dt.date.fromisoformat(due)).days if due else 0
138
+ except Exception:
139
+ days = 0
140
+ d['days_overdue'] = max(days, 0)
141
+ d['open_signed'] = d['amount_residual_signed']
142
+ by_partner.setdefault(pid, []).append(d)
143
+
144
+ rows = []
145
+ for p in partners:
146
+ pid = p['id']
147
+ odoo_overdue = float(p.get('total_overdue') or 0)
148
+ docs_p = sorted(by_partner.get(pid, []), key=lambda x: x.get('invoice_date_due') or '')
149
+ inv_overdue = sum(d['open_signed'] for d in docs_p if d['days_overdue'] > 0)
150
+ oldest = max((d['days_overdue'] for d in docs_p), default=0)
151
+ gap = inv_overdue - odoo_overdue
152
+ reconcile = abs(gap) > 50
153
+
154
+ if odoo_overdue >= 5000 or (odoo_overdue > 0 and oldest > 90):
155
+ tier = 'A-Urgent'
156
+ elif odoo_overdue >= 1000 or (odoo_overdue > 0 and oldest > 30):
157
+ tier = 'B-Active'
158
+ elif odoo_overdue > 0:
159
+ tier = 'C-Light'
160
+ else:
161
+ tier = 'Monitor'
162
+
163
+ rows.append({
164
+ 'partner_id': pid,
165
+ 'Customer': p['name'],
166
+ 'Tier': tier,
167
+ 'Overdue': odoo_overdue,
168
+ 'Receivable': float(p.get('credit') or 0),
169
+ 'Oldest (days)': oldest,
170
+ 'Open Docs': len(docs_p),
171
+ 'Email': p.get('email') or '',
172
+ 'Phone': p.get('phone') or p.get('mobile') or '',
173
+ 'Status': (p.get('followup_status') or '').replace('_', ' '),
174
+ 'Reconcile?': 'YES' if reconcile else '',
175
+ '_docs': docs_p,
176
+ })
177
+ tier_rank = {'A-Urgent': 0, 'B-Active': 1, 'C-Light': 2, 'Monitor': 3}
178
+ rows.sort(key=lambda r: (tier_rank[r['Tier']], -r['Overdue']))
179
+ return rows
180
+
181
+
182
+ # --------------------------------------------------------------- statement email
183
+ # ⛔ STANDING RULE 2, AND THIS IS THE ONE STRING IN THE PRODUCT THAT LEAVES THE BUILDING.
184
+ # Every other finding `web_prose` reports is copy on a screen somebody here opens; this is the
185
+ # SUBJECT LINE of mail queued to a real debtor, over the single sanctioned Odoo write. It read
186
+ # 'Statement of Account (em dash) {company} (em dash) {month}' until W36-T42.
187
+ DEFAULT_SUBJECT = 'Statement of Account from {company}, {month}'
188
+ DEFAULT_INTRO = (
189
+ 'Dear {customer},<br><br>'
190
+ 'Please find below your current statement of account with {company}. '
191
+ 'According to our records, the following invoices remain open:'
192
+ )
193
+ DEFAULT_FOOTER = (
194
+ 'If you have already sent payment, please disregard this notice, and thank you. '
195
+ 'For any questions about an invoice, simply reply to this email.<br><br>'
196
+ 'Thank you for your business,<br>{company}<br>Accounts Receivable'
197
+ )
198
+
199
+
200
+ def render_statement_html(row, intro_tpl=DEFAULT_INTRO, footer_tpl=DEFAULT_FOOTER):
201
+ month = dt.date.today().strftime('%B %Y')
202
+ intro = intro_tpl.format(customer=row['Customer'], company=COMPANY, month=month)
203
+ footer = footer_tpl.format(customer=row['Customer'], company=COMPANY, month=month)
204
+
205
+ lines = []
206
+ total_open = 0.0
207
+ for d in row['_docs']:
208
+ kind = 'Credit Note' if d['move_type'] == 'out_refund' else 'Invoice'
209
+ amt = d['open_signed']
210
+ total_open += amt
211
+ overdue_txt = f"{d['days_overdue']}d overdue" if d['days_overdue'] > 0 else 'current'
212
+ color = '#C0392B' if d['days_overdue'] > 0 else '#1F4E78'
213
+ lines.append(
214
+ f"<tr><td style='padding:6px 10px;border-bottom:1px solid #e3e8ef'>{d['name']} <span style='color:#888'>({kind})</span></td>"
215
+ f"<td style='padding:6px 10px;border-bottom:1px solid #e3e8ef'>{d.get('invoice_date') or ''}</td>"
216
+ f"<td style='padding:6px 10px;border-bottom:1px solid #e3e8ef'>{d.get('invoice_date_due') or ''}</td>"
217
+ f"<td style='padding:6px 10px;border-bottom:1px solid #e3e8ef;color:{color}'>{overdue_txt}</td>"
218
+ f"<td style='padding:6px 10px;border-bottom:1px solid #e3e8ef;text-align:right'>${amt:,.2f}</td></tr>")
219
+
220
+ table = (
221
+ "<table style='border-collapse:collapse;font-size:14px;margin:14px 0'>"
222
+ "<tr style='background:#1F4E78;color:#fff'>"
223
+ "<th style='padding:7px 10px;text-align:left'>Document</th>"
224
+ "<th style='padding:7px 10px;text-align:left'>Date</th>"
225
+ "<th style='padding:7px 10px;text-align:left'>Due</th>"
226
+ "<th style='padding:7px 10px;text-align:left'>Status</th>"
227
+ "<th style='padding:7px 10px;text-align:right'>Open Balance</th></tr>"
228
+ + ''.join(lines) +
229
+ f"<tr><td colspan='4' style='padding:8px 10px;font-weight:bold;text-align:right'>Total open</td>"
230
+ f"<td style='padding:8px 10px;font-weight:bold;text-align:right'>${total_open:,.2f}</td></tr>"
231
+ f"<tr><td colspan='4' style='padding:2px 10px;font-weight:bold;text-align:right;color:#C0392B'>Of which overdue</td>"
232
+ f"<td style='padding:2px 10px;font-weight:bold;text-align:right;color:#C0392B'>${row['Overdue']:,.2f}</td></tr>"
233
+ "</table>")
234
+
235
+ return (f"<div style='font-family:Calibri,Arial,sans-serif;color:#1a1a1a;font-size:14px'>"
236
+ f"{intro}{table}{footer}</div>")
237
+
238
+
239
+ def queue_statement(odoo, row, subject_tpl=DEFAULT_SUBJECT,
240
+ intro_tpl=DEFAULT_INTRO, footer_tpl=DEFAULT_FOOTER,
241
+ override_to=None):
242
+ """Queue one statement email in Odoo. Returns mail.mail id.
243
+ override_to: send to a different address (used by the test-send button)."""
244
+ to = override_to or row['Email']
245
+ if not to:
246
+ raise ValueError(f"{row['Customer']} has no email address")
247
+ # Hard guardrail — refuse any recipient outside the allow-list while SAFE_MODE is on.
248
+ if not safe_recipient_ok(to):
249
+ raise SafeModeBlocked(
250
+ f"Guardrail ON: refusing to email {to}. Only {', '.join(sorted(SAFE_RECIPIENTS))} "
251
+ f"is allowed right now. (Set SAFE_MODE=0 to send to real customers.)")
252
+ month = dt.date.today().strftime('%B %Y')
253
+ subject = subject_tpl.format(customer=row['Customer'], company=COMPANY, month=month)
254
+ payload = {
255
+ 'subject': subject,
256
+ 'body_html': render_statement_html(row, intro_tpl, footer_tpl),
257
+ 'email_to': to,
258
+ 'email_from': SENDER_DISPLAY, # From: Royal Imports
259
+ 'reply_to': REPLY_TO, # replies -> AR
260
+ 'mail_server_id': ROYAL_MAIL_SERVER_ID, # force Royal O365 relay
261
+ 'author_id': ROYAL_AUTHOR_ID, # clean attribution in chatter
262
+ 'state': 'outgoing',
263
+ 'auto_delete': False,
264
+ 'model': 'res.partner',
265
+ 'res_id': row['partner_id'],
266
+ }
267
+ return odoo.queue_mail(payload)
platform/modules/product_data.py CHANGED
The diff for this file is too large to render. See raw diff