diff --git a/RELEASES.json b/RELEASES.json index 15830ef7539e8e75614031bdad5891250ee1f1ea..f2bab27a950fe9faf48835fc5d624e80b34b2c80 100644 --- a/RELEASES.json +++ b/RELEASES.json @@ -1,6 +1,12 @@ { - "current": "624b9af", + "current": "v30 (71776ca)", "releases": [ + { + "version": "v30", + "sha": "71776ca", + "date": "2026-08-20", + "subject": "v30: W39-T29 Member-owned View first-share hotfix" + }, { "version": "v29", "sha": "55ced50", diff --git a/VERSION b/VERSION index f1045e5419bb6e867b4b59c7e4316662de22cb19..9db7eb0a25858e9592de675ebe22de39b50bf24b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -624b9af +v30 (71776ca) diff --git a/api/ai_review.py b/api/ai_review.py index 093c323357f15f25b5301e2db540196f9d7855ee..03f68bd48c54f4543883f221926d84a4cd135c1f 100644 --- a/api/ai_review.py +++ b/api/ai_review.py @@ -1,591 +1,591 @@ -"""AI REVIEW — let a model decide which stage a card moves to (wave 23, owner ruling R4/R14). - -⭐ WHAT THIS IS. A review stage holds a record until somebody decides where it goes next. R4 made -that somebody optionally a MODEL: the engine hands over the record's own values, the review's -prompt, and the list of stages the review is allowed to send a card to, and gets back ONE of -those stage labels plus a one-line reason. Every decision is written to the same `reviews` audit -log a human click writes to, tagged `by: "ai"` with the provider and model that made it. - -⛔ FAIL-CLOSED IN EVERY DIRECTION, and this is the whole safety story. No key configured, a -network failure, a slow answer, a malformed answer, or an answer naming a stage the review does -not offer — all return `("", {...})`, and the caller leaves the card exactly where a human would -have found it. The feature can be absent, broken or wrong and the worst outcome is a person doing -the work. Nothing here can move a card somewhere the review does not already permit. - -⭐ CHEAP FIRST (owner R14, verbatim: *"Claude is a bit too expensive"*). The ladder is -`groq → cerebras → openrouter → anthropic`; the first CONFIGURED provider wins, and Anthropic is -last rather than absent — it is the quality backstop, not the default. `AIOS_AI_REVIEW_PROVIDER` -pins one; `AIOS_AI_MODEL` overrides the model. - -⭐⭐ THE ANTHROPIC LEG IS ON THE OFFICIAL SDK NOW (D-346, W37-T39, 2026-08-19). ⛔ THIS PARAGRAPH -USED TO ARGUE THE OPPOSITE and is rewritten rather than deleted, because a comment left asserting -the reverse of its own code is [[two-gates-can-assert-opposite-things]] with no gate to catch it. -The retired text said raw HTTP was *"a deliberate deviation … booked as a DEBT line so the -integrator can overturn it deliberately rather than by drift"*. This is that overturn. - -Its two reasons were real and are both answered rather than waved away: - 1. *"one SDK leg beside three hand-rolled ones is two implementations of the same call."* It is - not, because the leg does not live here: `providers.anthropic_send` owns the wire, prefers - the SDK, and normalises BOTH transports to one `(status, body)` pair. This file gained no - Anthropic knowledge — it lost some. - 2. *"adding a pinned dependency rebuilds the container, a real deploy risk."* Still true, so the - import is LAZY and its absence is a FALLBACK, not a crash: with no `anthropic` package the - same raw POST runs and `LAST_ANTHROPIC_TRANSPORT` reports `'http'`. The pin is owed in BOTH - manifests ([[pin-deps-space-rebuilds]]) and is outside every worker fence this wave. -⚠ The other three legs stay OpenAI-chat-shaped over `requests` — they have no shared SDK, and that -half of the original reasoning never expired. - -The Messages shape below is the current one: `x-api-key` + `anthropic-version: 2023-06-01`, and -`stop_reason: "refusal"` is checked BEFORE reading `content` — a refusal answers HTTP 200 with an -empty content list, so code that indexes `content[0]` unconditionally breaks on it. -""" -from __future__ import annotations - -import json -import os -import re - -import requests - -#: The ladder. Order IS the policy (R14) — cheapest capable first, Anthropic last. -PROVIDERS = [ - # ⛔⛔ D-345, FIXED W37-T39 (2026-08-19). This read `llama-3.3-70b-versatile` and Groq RETIRED - # it: a real POST with the live key answers HTTP 404 `model_not_found`, and the id is absent - # from `GET /openai/v1/models`. ⚠ THE `openai/` PREFIX IS PART OF GROQ'S ID and is NOT a typo - # for the cerebras rung below, which serves the same family bare as `gpt-oss-120b`. - {"name": "groq", "env": "GROQ_API_KEY", "shape": "openai", - "url": "https://api.groq.com/openai/v1/chat/completions", - "model": "openai/gpt-oss-120b"}, - {"name": "cerebras", "env": "CEREBRAS_API_KEY", "shape": "openai", - "url": "https://api.cerebras.ai/v1/chat/completions", - "model": "gpt-oss-120b"}, - {"name": "openrouter", "env": "OPENROUTER_API_KEY", "shape": "openai", - "url": "https://openrouter.ai/api/v1/chat/completions", - "model": "openai/gpt-4o-mini"}, - # ⚠ haiku-class DELIBERATELY, not the default Opus tier: R14 put Anthropic on this ladder as - # the backstop for a one-line classification, and this is the cheapest current Claude that - # does it well. A bigger model here would be spending the owner's money to pick between two - # labels it already has in front of it. - {"name": "anthropic", "env": "ANTHROPIC_API_KEY", "shape": "anthropic", - "url": "https://api.anthropic.com/v1/messages", - # ⚠ AN ENV LEVER, NOT A NEW DEFAULT (W36-T35): haiku stays the choice for the reasons above, - # and a deployment whose side rungs are out of credit can raise the tier without a release. - "model": os.environ.get("AIOS_AI_REVIEW_ANTHROPIC_MODEL") or "claude-haiku-4-5"}, -] -ANTHROPIC_VERSION = "2023-06-01" -TIMEOUT_SECONDS = float(os.environ.get("AIOS_AI_REVIEW_TIMEOUT") or 20) -MAX_FIELD_CHARS = 200 # per value handed to the model -MAX_FIELDS = 30 # columns handed to the model -MAX_REASON = 200 - - -def ladder(): - """The providers that are actually usable here, in order. Empty = the feature is off.""" - pin = (os.environ.get("AIOS_AI_REVIEW_PROVIDER") or "").strip().lower() - live = [p for p in PROVIDERS if (os.environ.get(p["env"]) or "").strip()] - if pin: - live = [p for p in live if p["name"] == pin] - return live - - -def configured(): - return bool(ladder()) - - -def _record_text(row, fields): - """The record, as the model sees it. Values are truncated and the column set is bounded — - an automation table can carry a 32 KB JSON blob per row (C7) and a review decision does not - need it. Machine bookkeeping columns are dropped: a stage cell naming the stage the card is - sitting at would be the model reading its own question back.""" - keys = [k for k in (fields or list((row or {}).keys())) - if not str(k).startswith("stage_")][:MAX_FIELDS] - lines = [] - for k in keys: - v = (row or {}).get(k) - if v is None or str(v).strip() == "": - continue - lines.append(f"{k}: {str(v)[:MAX_FIELD_CHARS]}") - return "\n".join(lines) or "(this record has no filled-in values)" - - -def _instruction(prompt, options, label): - return ( - f"You are deciding what happens to one record waiting at a review step called " - f"{label!r} in a workflow.\n\n" - f"The person who built this workflow told you: {prompt}\n\n" - f"Choose EXACTLY ONE of these next steps, by its exact name:\n" - + "\n".join(f"- {o}" for o in options) - + "\n\nAnswer with one line of JSON and nothing else:\n" - '{"choice": "", "reason": ""}\n' - "If the record does not give you enough to decide, answer " - '{"choice": "", "reason": "why not"} and a person will decide instead.' - ) - - -def _parse(text, options): - """The model's line → `(choice, reason)`. A choice that is not one of the offered stages is - DISCARDED, not fuzzy-matched: the offered list is a permission boundary, and a near-miss - resolved by string distance is how a card ends up somewhere nobody authorised.""" - raw = str(text or "").strip() - obj = None - m = re.search(r"\{.*\}", raw, re.S) - if m: - try: - obj = json.loads(m.group(0)) - except (ValueError, TypeError): - obj = None - if not isinstance(obj, dict): - return "", "" - choice = str(obj.get("choice") or "").strip() - reason = str(obj.get("reason") or "").strip()[:MAX_REASON] - for opt in options: - if choice.lower() == str(opt).lower(): - return str(opt), reason # the OFFERED spelling wins, never the model's - return "", reason - - -# ⭐⭐ WAVE 35 · W35-T41 / C7 — THE TWO TRANSPORTS RETURN THE RESPONSE BODY'S `usage`. -# -# ⛔ THIS FILE'S OWN HEADER USED TO BE THE PRODUCT'S CONFESSION THAT NOTHING COUNTED: `ai_enrich`'s -# docstring says *"`ai_review.decide` -- the product's only other LLM entry -- has no token -# accounting of ANY kind"*. R9 asks for ONE meter for every AI surface, so the counting has to reach -# this transport rather than be bolted onto its caller — the caller never sees the body. -# -# ⚠ A THIRD RETURN VALUE, NOT A MUTATED ARGUMENT, and both call sites are in `decide` below. The -# tuple grew from `(text, err)` to `(text, err, usage)`; `usage` is the raw `usage` OBJECT (or None), -# because reading it is `usage_ledger`'s job and a second reader here would be the two-normalizers -# shape this repo keeps paying for. -def _call_openai(p, model, system, user, timeout): - r = requests.post(p["url"], timeout=timeout, - headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}", - "Content-Type": "application/json"}, - json={"model": model, "max_tokens": 300, "temperature": 0, - "messages": [{"role": "system", "content": system}, - {"role": "user", "content": user}]}) - if r.status_code >= 400: - # ⚠ NO BODY ON A 4xx/5xx: an error envelope carries no usage, and a provider that refused - # before running the model has nothing to bill. The call is still COUNTED by `decide`. - return "", f"{p['name']} answered {r.status_code}", None - body = r.json() - choices = body.get("choices") or [] - if not choices: - return "", f"{p['name']} returned no choices", body - return str(((choices[0] or {}).get("message") or {}).get("content") or ""), "", body - - -#: What transport the last Anthropic call actually used: `'sdk'`, `'http'`, or `''` before any. -#: ⚠ PROCESS-LOCAL AND FOR REPORTING ONLY. It exists so `GET /meta` and `verify_web_agent` can say -#: WHICH path ran rather than inferring it from a requirements file nobody in this lane can edit. -LAST_ANTHROPIC_TRANSPORT = "" - - -def _call_anthropic(p, model, system, user, timeout): - """One classification turn on the Messages API, through `providers.anthropic_send`. - - ⭐ D-346 (W37-T39): this used to hand-roll the POST. It now goes through the shared wire, which - prefers the OFFICIAL `anthropic` SDK and falls back to the same raw POST when the package is - absent. The shape of what comes back is unchanged, deliberately: `anthropic_send` normalises - both transports to `(status, body)`, so every line below this call is untouched. - """ - global LAST_ANTHROPIC_TRANSPORT - import providers as _prov # noqa: PLC0415 - # ⚠ NO TOOLS ON THIS PATH. The builder omits `tool_choice` for exactly that case, so there is - # nothing to strip here — the rule lives in `anthropic_request`, beside the thing it constrains. - req = _prov.anthropic_request( - model=model, key=os.environ[p["env"]].strip(), system=system, - messages=[{"role": "user", "content": user}], tools=[], max_tokens=300) - status, body, transport = _prov.anthropic_send(req, timeout=timeout) - LAST_ANTHROPIC_TRANSPORT = transport - if status >= 400 or status == 0: - # ⭐ THE SENTENCE COMES FROM THE LADDER, NOT FROM HERE. `refusal_sentence` already names the - # vendor and the action for every status R4 enumerated, so a bare "anthropic answered 402" - # (which is what this line used to say, and what the owner quoted back at us) cannot recur. - return "", _prov.refusal_sentence("anthropic", status, json.dumps(body)[:400]), None - body = body or {} - # ⛔ stop_reason FIRST. A safety refusal is a successful 200 with an EMPTY content list, so - # reading content[0] before this check turns a refusal into an IndexError inside a run. - # ⚠ A refusal IS billed and its body carries `usage`, so the body rides back on this branch too. - if body.get("stop_reason") == "refusal": - return "", "anthropic declined to answer this record", body - parts = [b.get("text") or "" for b in (body.get("content") or []) - if isinstance(b, dict) and b.get("type") == "text"] - if not parts: - return "", "anthropic returned no text", body - return "".join(parts), "", body - - -def decide(*, prompt, options, row, fields=(), label="Review", timeout=None, st=None, user=""): - """Pick this record's next stage. Returns `(choice, meta)`. - - `choice` is "" whenever a person should decide — which is every failure mode there is. - `meta` carries `provider`, `model`, `reason` on success, and `problem` on refusal to answer. - - ⭐ W35-T41 / C7 — `st` and `user` are the USAGE LEDGER's target. Both default to absent because - this function's caller is `automation_engine.ai_decide(rt, ...)`, in another lane's fence: it - HAS the runtime and does not pass it yet, so until it does, a review's tokens are counted as - unattributed and REPORTED by `GET /usage` rather than dropped. See `usage_ledger`'s header for - why they cannot be resolved implicitly (measured: a contextvar does not survive a FastAPI - dependency). - """ - import usage_ledger # noqa: PLC0415 - opts = [str(o) for o in (options or []) if str(o).strip()] - if not opts: - return "", {"problem": "the review offers no next stages"} - if not str(prompt or "").strip(): - return "", {"problem": "the review has no prompt for the model to follow"} - live = ladder() - if not live: - return "", {"problem": "no AI provider is configured on this deployment"} - system = _instruction(prompt, opts, label) - user = "Here is the record:\n\n" + _record_text(row, fields) - tmo = float(timeout or TIMEOUT_SECONDS) - override = (os.environ.get("AIOS_AI_MODEL") or "").strip() - problems = [] - for p in live: - model = override or p["model"] - body = None - try: - text, err, body = (_call_anthropic if p["shape"] == "anthropic" else _call_openai)( - p, model, system, user, tmo) - except Exception as e: # noqa: BLE001 - text, err = "", f"{p['name']} failed: {type(e).__name__}" - # ⭐⭐ C7 — THE LEDGER LINE, BEFORE ANY BRANCH BELOW READS THE ANSWER. - # ⛔ IT IS WRITTEN ON EVERY OUTCOME THAT REACHED A PROVIDER, including a refusal and an - # unusable answer. A meter that books only successes reports a cheap week for a run that - # spent its budget being declined — and a declined call is billed. The only path that does - # NOT record is a transport that never reached the vendor (`body is None`), which spent - # nothing. - if body is not None: - ins, outs = usage_ledger.tokens_from(body) - usage_ledger.record("ai_review", p["name"], model, ins, outs, - total=usage_ledger.total_from(body), st=st, user=user) - if err: - problems.append(err) - continue # ladder: a dead provider degrades to the next one - choice, reason = _parse(text, opts) - if not choice: - # The provider ANSWERED and declined (or answered unusably). That is a decision about - # this record, not a fault in the provider, so it does NOT fall through to a more - # expensive one — the card goes to a human, which is what the model just asked for. - return "", {"provider": p["name"], "model": model, - "problem": reason or "the model did not choose one of the stages"} - return choice, {"provider": p["name"], "model": model, "reason": reason} - return "", {"problem": "; ".join(problems)[:300] or "no provider answered"} - - -# ══════════════════════════════════════════════════════════════════ the FLOW WRITER -# ⭐⭐ WAVE 33 · W33-T54/T55 (owner item 7, ruling R3) — A PROMPT BECOMES A DRAFT AUTOMATION. -# -# Owner, verbatim: *"lay out the foundation with custom tools we provide"* — and the tools are the -# action catalog, which has been machine-readable since wave 23. So this writes no new vocabulary: -# it hands the model the SAME `ACTION_CATALOG` the menu paints, the SAME `ACTION_REQUIRED` the -# runner blocks on, and the SAME trigger keys `clean_trigger` accepts, and asks for one JSON object -# in that vocabulary. -# -# ⛔ WHY IT LIVES IN `ai_review.py` RATHER THAN A NEW FILE. Two reasons, both about ownership. This -# is the one module in the API package that already holds an LLM ladder and its credentials, so a -# second one would be a second place a rotated key has to be noticed; and a new `routes_*.py` needs -# a `main.py` line from another lane (contract C2) to be reachable at all, which is how wave 23 -# shipped three finished routers 404-dead behind green gates. The DOOR is a route on -# `routes_automation.py`, which is already mounted. -# -# ⛔⛔ CONTRACT C7 IS THE DESIGN, NOT A CHECK AT THE END: *"the flow writer may emit only kinds -# present in `ACTION_CATALOG`, and its output must pass `clean_actions` unchanged."* Both halves are -# enforced mechanically rather than asked for politely — the kind list is an `enum` in the schema -# the model fills, and the caller runs `clean_actions` and DIFFS the result. That diff matters -# because `clean_actions` has no disclosure channel (D-75): it drops a key it does not recognise -# and answers 200, so a draft accepted without the diff would show a person a flow that is not the -# flow the model described, with nothing anywhere saying so. -# -# ⚠ `chat` IS INJECTABLE, exactly as `routes_query._call_model`'s is, and it is the reason this door -# can be proven end to end with no API key and no spend. A gate that can only run where a -# credential exists is a gate that never runs. - -#: Cerebras first, NOT the cheap-first order `ladder()` uses. Same reason `routes_query` inverts it: -#: this contract includes a REFUSAL, and refusing honestly ("I cannot build that from the steps you -#: have") is a model property. A cheaper rung answers an impossible request with a plausible flow, -#: which is worse than no flow. Override with `AIOS_FLOW_PROVIDER`. -FLOW_PROVIDER_ORDER = ("cerebras", "groq", "openrouter", "anthropic") -MAX_DRAFT_ACTIONS = 12 # a draft a person reads in one screen; the engine's own cap is 25 -MAX_PROMPT_CHARS = 2000 - - -def flow_providers(pin=None): - """The rungs usable here, in THIS module's refusal-first order. Empty = the feature is off. - - ⭐ `pin` IS ASK D-18 (2026-08-18): the Agent chat's model toggle must configure something, and - the draft door used to read `prompt` off the body and nothing else — so the key the client sent - was accepted and dropped, and the picker was a control over nothing. - ⚠ AN UNKNOWN OR UNCONFIGURED PIN FALLS BACK TO THE LADDER rather than refusing. A model the - ladder stopped offering must not turn every later draft into an error; the caller is told which - rung actually answered, which is the honest half. - """ - by_name = {p["name"]: p for p in PROVIDERS} - live = [n for n in FLOW_PROVIDER_ORDER - if n in by_name and (os.environ.get(by_name[n]["env"]) or "").strip()] - wanted = str(pin or os.environ.get("AIOS_FLOW_PROVIDER") or "").strip().lower() - if wanted and wanted in live: - return [by_name[wanted]] - return [by_name[n] for n in live] - - -def flow_schema(kinds, trigger_keys, table_keys): - """The tool schema the model fills — the catalog's OWN key lists as enums. - - ⛔ `kind` AND `trigger` ARE ENUMS, not free strings, and that is C7's first half enforced by the - transport rather than by a check afterwards. A model asked for "any action name" invents - `send_slack_message` on a deployment that has no such kind, and the failure then surfaces as a - 400 from `clean_actions` carrying a sentence about a word the person never typed. - ⚠ `required` IS `["kind"]` ALONE, deliberately: a REFUSAL carries no name and no actions, and a - schema demanding them turns an honest refusal into a provider-side 400 that reads exactly like a - transport failure (`routes_query._spec_schema` carries the same note, for the same measured - reason). - """ - return { - "type": "object", - "properties": { - "kind": {"type": "string", "enum": ["flow", "refused"], - "description": "refused = this cannot be built from the steps available"}, - "refusal": {"type": "string", - "description": "when kind=refused: ONE plain sentence naming what is " - "missing, in the words a non-technical person would use"}, - "name": {"type": "string", "description": "a short title for the automation"}, - "trigger": {"type": "string", "enum": sorted(trigger_keys), - "description": "what starts this automation"}, - "table": {"type": "string", "enum": sorted(table_keys), - "description": "the database whose records this flow walks, if any"}, - "actions": { - "type": "array", - "maxItems": MAX_DRAFT_ACTIONS, - "items": { - "type": "object", - "properties": { - "kind": {"type": "string", "enum": sorted(kinds)}, - "why": {"type": "string", - "description": "one short sentence: why this step is here"}, - "config": {"type": "object", "additionalProperties": True, - "description": "the step's settings, using ONLY the keys named " - "for that kind in the system message"}, - }, - "required": ["kind"], - }, - }, - }, - "required": ["kind"], - } - - -def flow_system_prompt(catalog, required, triggers, tables): - """What the model is told it may build with — DERIVED, never written down twice. - - Every list here is the server's own: the catalog rows the menu paints, the required-key table - the runner blocks on, the trigger keys `clean_trigger` accepts, and this tenant's real databases - with their real columns. Nothing about the vocabulary is restated by hand, so a kind added to - the catalog is offered here on the same deploy and a kind removed stops being offered. - """ - lines = ["You build small automations for a business tool. You are given the EXACT set of " - "steps this tool can perform. You may use nothing else.", - "", "THE STEPS YOU MAY USE:"] - # ⚠ BOTH HALVES, `key (phrase)`. The KEY is what the model must write into `config` and the - # PHRASE is the only human wording of that requirement anywhere — the one the runner's own - # refusal sentence is built from. Keys alone leave the model guessing what `field` means on a - # `web_read` (it is a column to write into, not a form field); phrases alone leave it guessing - # what to call the setting. Handing it one and hoping is how a draft comes back configured - # against a key the validator drops. - req = {str(k): [f"{key} ({phrase})" for phrase, key in v] for k, v in (required or {}).items()} - for row in catalog or []: - if not row.get("ready"): - continue - kind = str(row.get("kind") or "") - need = req.get(kind) or [] - lines.append(f"- {kind}: {row.get('label')} - {row.get('detail') or ''}" - + (f" REQUIRED settings: {', '.join(need)}" if need else "")) - lines += ["", "WHAT CAN START AN AUTOMATION:"] - for t in triggers or []: - if t.get("planned") or not t.get("ready", True): - continue - lines.append(f"- {t.get('key')}: {t.get('label')}") - lines += ["", "THE DATABASES THIS PERSON HAS:"] - for t in (tables or [])[:40]: - cols = ", ".join(str(f.get("key")) for f in (t.get("fields") or [])[:25]) - lines.append(f"- {t.get('key')} ({t.get('label')}): {cols or 'no columns yet'}") - if not tables: - lines.append("- (none - do not name a database)") - lines += [ - "", - "RULES:", - "1. Use ONLY the step kinds listed above. If what is asked needs a step that is not there, " - "answer kind=refused and say plainly which capability is missing.", - "2. Fill in every REQUIRED setting you can from what the person told you. Leave one blank " - "rather than inventing a web address, a CSS selector or a column name.", - "3. A web address must start with http:// or https://.", - "4. To use a value from the record the flow is walking, write it as {{Column name}}.", - "5. Name a database only from the list above, by its key.", - "6. Keep it short. Fewer steps that work beat more steps that guess.", - ] - return "\n".join(lines) - - -def _flow_from_tool_call(obj): - """The model's tool arguments -> `(draft, refusal)`. A shape error is a refusal, never a crash.""" - if not isinstance(obj, dict): - return None, "the assistant's answer could not be read" - if str(obj.get("kind") or "") == "refused": - return None, (str(obj.get("refusal") or "").strip() - or "the assistant could not build this from the steps available") - acts = obj.get("actions") - if not isinstance(acts, list) or not acts: - return None, ("the assistant did not produce any steps - try describing what should " - "happen, one action at a time") - out = [] - for a in acts[:MAX_DRAFT_ACTIONS]: - if not isinstance(a, dict) or not str(a.get("kind") or "").strip(): - continue - cfg = a.get("config") - out.append({"kind": str(a["kind"]).strip(), - "config": cfg if isinstance(cfg, dict) else {}, - "why": str(a.get("why") or "").strip()[:200]}) - if not out: - return None, "the assistant's steps could not be read" - return {"name": str(obj.get("name") or "").strip()[:80] or "New automation", - "trigger": str(obj.get("trigger") or "").strip(), - "table": str(obj.get("table") or "").strip(), - # ⛔⛔ THE TRUNCATION IS REPORTED, and it is a STANDING RULE that it must be (owner, - # 2026-08-12, W30/R6 second sentence): *"if there is lag or it can't be done, you need - # to explicitly tell me why and recommend a fix"* — **a silent truncation IS the - # violation, not the limit.** `acts[:MAX_DRAFT_ACTIONS]` above dropped everything past - # the twelfth and said nothing, so a model that answered with a twenty-step journey had - # eight steps deleted between the answer and the screen, with no key anywhere in the - # response naming them. `MAX_ACTIONS` is 20, so those steps were STORABLE — this - # ceiling is the draft door's own, which makes reporting it the whole obligation. - # ⚠ Found by the verifier that checked this ticket, not by a gate. - "asked": len([a for a in acts if isinstance(a, dict)]), - "actions": out}, "" - - -def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeout=None, - st=None, user="", model=None): - """A sentence -> `(draft, refusal_sentence, provider)`. ⛔ NOTHING IS SAVED HERE. - - Exactly one of `draft` and `refusal_sentence` is truthy — the same contract - `web_agent.run_step` keeps, so a caller has no third case to get wrong. - - ⭐ W35-T41 / C7 — `st`/`user` are the usage ledger's target, exactly as on `decide` above and for - the same reason: both of this function's callers (`automation_engine._ai_agent_plan` and - `routes_automation`'s draft door) are in lane D's fence. C7 says E adds the ledger line here and - D asserts it; the two keywords are what D has to pass for the line to be attributable. - """ - # ⚠ DECLARED AT THE TOP OF THE FUNCTION, not beside the assignment inside the provider loop. - # It parses either way; a reader scanning for the declaration does not look inside a `for`. - global LAST_ANTHROPIC_TRANSPORT - import usage_ledger # noqa: PLC0415 - text = str(prompt or "").strip()[:MAX_PROMPT_CHARS] - if not text: - return None, "type what you want the automation to do", None - kinds = sorted({str(r.get("kind")) for r in (catalog or []) if r.get("ready")}) - if not kinds: - return None, "this deployment offers no automation steps to build with", None - trigger_keys = sorted({str(t.get("key")) for t in (triggers or []) - if t.get("key") and not t.get("planned")}) or ["manual"] - table_keys = sorted({str(t.get("key")) for t in (tables or []) if t.get("key")}) or [""] - tools = [{"type": "function", "function": { - "name": "build_automation", - "description": "Emit the automation, or refuse.", - "parameters": flow_schema(kinds, trigger_keys, table_keys)}}] - messages = [{"role": "system", - "content": flow_system_prompt(catalog, required, triggers, tables)}, - {"role": "user", "content": text}] - - if chat is not None: - # ⚠ THE INJECTED PATH IS THE PROVEN PATH. It runs the SAME parse and the SAME refusal - # branches as a live call; only the transport is replaced. - draft, refusal = _flow_from_tool_call(chat(messages, tools)) - return draft, refusal, "injected" - - provs = flow_providers(model) - if not provs: - # ⛔ SAY SO. An AI feature that silently does nothing is indistinguishable from one that was - # never built [[flag-shipped-without-its-writer]]. - return None, ("the assistant is not configured on this deployment, so an automation cannot " - "be drafted from a description yet"), None - tmo = float(timeout or TIMEOUT_SECONDS) - problems = [] - import providers as _prov - for p in provs: - # ⭐⭐ W36-T35 / R4 — THE OWNER QUOTED THIS LINE BACK AT US. It used to read - # `problems.append(f"{p['name']}: tool calls are not wired for this shape")` and `continue`, - # so the sentence *"anthropic: tool calls are not wired for this shape"* appeared under the - # Agent module verbatim. R4: *"Anthropic becomes the tool-calling path that always works."* - # It is wired now, through the ONE wire in `providers`, and it matters more than it looks: - # every side rung on this account is refusing today (cerebras 402, groq 404, openrouter - # 402), so without this branch the drafter cannot answer at all. - # ⚠ NO `effort` ON THIS DOOR — `output_config.effort` errors on Haiku 4.5, the tier this - # ladder runs. The parameter is the caller's to send, which is why the wire takes it. - # ⭐ D-346 (W37-T39): BOTH shapes now yield the SAME `(status, body)` pair, so every branch - # below reads one thing. The anthropic side gets there through `anthropic_send` (official - # SDK when installed, the identical raw POST when not); the openai side still posts here - # because there is no shared SDK across three different vendors on that wire. - global LAST_ANTHROPIC_TRANSPORT - try: - if p["shape"] == "anthropic": - req = _prov.anthropic_request( - model=p["model"], key=os.environ[p["env"]].strip(), system=None, - messages=messages, tools=tools, max_tokens=1500, tool_choice="required") - status, body, transport = _prov.anthropic_send(req, timeout=tmo) - LAST_ANTHROPIC_TRANSPORT = transport - else: - r = requests.post(p["url"], timeout=tmo, - headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}", - "Content-Type": "application/json"}, - json={"model": p["model"], "messages": messages, "tools": tools, - "tool_choice": "required", "temperature": 0.1, - "max_tokens": 1500}) - status = r.status_code - body = r.json() if (r.content and r.status_code == 200) else {"_text": r.text} - except Exception as e: # noqa: BLE001 - problems.append(f"{p['name']}: {type(e).__name__}") - continue - if status != 200: - # ⛔ A SENTENCE, NOT `HTTP 402` (R4's third clause). The owner read the status codes off - # this very door. `refusal_sentence` also decides whether this was a CREDIT failure, and - # the memo makes the next turn SKIP the empty account instead of paying for it again. - # ⚠ The body is re-serialised for the substring test because the SDK path never had a - # `.text` — that is the one thing this normalisation costs, and it costs nothing else. - detail = json.dumps(body)[:600] if isinstance(body, dict) else str(body)[:600] - if _prov.is_credit_failure(status, detail): - _prov.mark_no_credit(p["name"]) - problems.append(_prov.refusal_sentence(p["name"], status, detail)) - continue - try: - body = body or {} - if p["shape"] == "anthropic": - _text, args, _refused = _prov.anthropic_read(body) - if _refused: - problems.append(f"{p['name']}: {_refused}") - continue - else: - calls = (((body.get("choices") or [{}])[0].get("message") or {}) - .get("tool_calls") or []) - args = json.loads(calls[0]["function"]["arguments"]) if calls else None - except Exception as e: # noqa: BLE001 - problems.append(f"{p['name']}: unreadable answer ({type(e).__name__})") - continue - # ⭐⭐ C7 — THE LEDGER LINE. Placed after the parse rather than before it so an UNREADABLE - # answer is not double-counted by the `continue` above... ⚠ which means an unreadable answer - # is NOT counted at all, and that is a deliberate, disclosed loss: a body this code cannot - # parse is a body whose `usage` it also cannot trust, and `body` is out of scope in that - # branch by construction. The 200-with-junk case is rare and named here rather than - # silently rounded to zero. - ins, outs = usage_ledger.tokens_from(body) - usage_ledger.record("automation_draft", p["name"], p["model"], ins, outs, - total=usage_ledger.total_from(body), st=st, user=user) - draft, refusal = _flow_from_tool_call(args) - if draft is None and isinstance(args, dict) and str(args.get("kind")) == "refused": - # ⛔ A REFUSAL IS AN ANSWER, NOT A FAULT, so it does NOT fall through to a more - # expensive rung — the model has just said this cannot be built. `decide()` takes the - # same posture for the same reason. - return None, refusal, p["name"] - if draft is not None: - return draft, "", p["name"] - problems.append(f"{p['name']}: {refusal}") - return None, ("; ".join(problems)[:300] or "no provider answered"), None +"""AI REVIEW — let a model decide which stage a card moves to (wave 23, owner ruling R4/R14). + +⭐ WHAT THIS IS. A review stage holds a record until somebody decides where it goes next. R4 made +that somebody optionally a MODEL: the engine hands over the record's own values, the review's +prompt, and the list of stages the review is allowed to send a card to, and gets back ONE of +those stage labels plus a one-line reason. Every decision is written to the same `reviews` audit +log a human click writes to, tagged `by: "ai"` with the provider and model that made it. + +⛔ FAIL-CLOSED IN EVERY DIRECTION, and this is the whole safety story. No key configured, a +network failure, a slow answer, a malformed answer, or an answer naming a stage the review does +not offer — all return `("", {...})`, and the caller leaves the card exactly where a human would +have found it. The feature can be absent, broken or wrong and the worst outcome is a person doing +the work. Nothing here can move a card somewhere the review does not already permit. + +⭐ CHEAP FIRST (owner R14, verbatim: *"Claude is a bit too expensive"*). The ladder is +`groq → cerebras → openrouter → anthropic`; the first CONFIGURED provider wins, and Anthropic is +last rather than absent — it is the quality backstop, not the default. `AIOS_AI_REVIEW_PROVIDER` +pins one; `AIOS_AI_MODEL` overrides the model. + +⭐⭐ THE ANTHROPIC LEG IS ON THE OFFICIAL SDK NOW (D-346, W37-T39, 2026-08-19). ⛔ THIS PARAGRAPH +USED TO ARGUE THE OPPOSITE and is rewritten rather than deleted, because a comment left asserting +the reverse of its own code is [[two-gates-can-assert-opposite-things]] with no gate to catch it. +The retired text said raw HTTP was *"a deliberate deviation … booked as a DEBT line so the +integrator can overturn it deliberately rather than by drift"*. This is that overturn. + +Its two reasons were real and are both answered rather than waved away: + 1. *"one SDK leg beside three hand-rolled ones is two implementations of the same call."* It is + not, because the leg does not live here: `providers.anthropic_send` owns the wire, prefers + the SDK, and normalises BOTH transports to one `(status, body)` pair. This file gained no + Anthropic knowledge — it lost some. + 2. *"adding a pinned dependency rebuilds the container, a real deploy risk."* Still true, so the + import is LAZY and its absence is a FALLBACK, not a crash: with no `anthropic` package the + same raw POST runs and `LAST_ANTHROPIC_TRANSPORT` reports `'http'`. The pin is owed in BOTH + manifests ([[pin-deps-space-rebuilds]]) and is outside every worker fence this wave. +⚠ The other three legs stay OpenAI-chat-shaped over `requests` — they have no shared SDK, and that +half of the original reasoning never expired. + +The Messages shape below is the current one: `x-api-key` + `anthropic-version: 2023-06-01`, and +`stop_reason: "refusal"` is checked BEFORE reading `content` — a refusal answers HTTP 200 with an +empty content list, so code that indexes `content[0]` unconditionally breaks on it. +""" +from __future__ import annotations + +import json +import os +import re + +import requests + +#: The ladder. Order IS the policy (R14) — cheapest capable first, Anthropic last. +PROVIDERS = [ + # ⛔⛔ D-345, FIXED W37-T39 (2026-08-19). This read `llama-3.3-70b-versatile` and Groq RETIRED + # it: a real POST with the live key answers HTTP 404 `model_not_found`, and the id is absent + # from `GET /openai/v1/models`. ⚠ THE `openai/` PREFIX IS PART OF GROQ'S ID and is NOT a typo + # for the cerebras rung below, which serves the same family bare as `gpt-oss-120b`. + {"name": "groq", "env": "GROQ_API_KEY", "shape": "openai", + "url": "https://api.groq.com/openai/v1/chat/completions", + "model": "openai/gpt-oss-120b"}, + {"name": "cerebras", "env": "CEREBRAS_API_KEY", "shape": "openai", + "url": "https://api.cerebras.ai/v1/chat/completions", + "model": "gpt-oss-120b"}, + {"name": "openrouter", "env": "OPENROUTER_API_KEY", "shape": "openai", + "url": "https://openrouter.ai/api/v1/chat/completions", + "model": "openai/gpt-4o-mini"}, + # ⚠ haiku-class DELIBERATELY, not the default Opus tier: R14 put Anthropic on this ladder as + # the backstop for a one-line classification, and this is the cheapest current Claude that + # does it well. A bigger model here would be spending the owner's money to pick between two + # labels it already has in front of it. + {"name": "anthropic", "env": "ANTHROPIC_API_KEY", "shape": "anthropic", + "url": "https://api.anthropic.com/v1/messages", + # ⚠ AN ENV LEVER, NOT A NEW DEFAULT (W36-T35): haiku stays the choice for the reasons above, + # and a deployment whose side rungs are out of credit can raise the tier without a release. + "model": os.environ.get("AIOS_AI_REVIEW_ANTHROPIC_MODEL") or "claude-haiku-4-5"}, +] +ANTHROPIC_VERSION = "2023-06-01" +TIMEOUT_SECONDS = float(os.environ.get("AIOS_AI_REVIEW_TIMEOUT") or 20) +MAX_FIELD_CHARS = 200 # per value handed to the model +MAX_FIELDS = 30 # columns handed to the model +MAX_REASON = 200 + + +def ladder(): + """The providers that are actually usable here, in order. Empty = the feature is off.""" + pin = (os.environ.get("AIOS_AI_REVIEW_PROVIDER") or "").strip().lower() + live = [p for p in PROVIDERS if (os.environ.get(p["env"]) or "").strip()] + if pin: + live = [p for p in live if p["name"] == pin] + return live + + +def configured(): + return bool(ladder()) + + +def _record_text(row, fields): + """The record, as the model sees it. Values are truncated and the column set is bounded — + an automation table can carry a 32 KB JSON blob per row (C7) and a review decision does not + need it. Machine bookkeeping columns are dropped: a stage cell naming the stage the card is + sitting at would be the model reading its own question back.""" + keys = [k for k in (fields or list((row or {}).keys())) + if not str(k).startswith("stage_")][:MAX_FIELDS] + lines = [] + for k in keys: + v = (row or {}).get(k) + if v is None or str(v).strip() == "": + continue + lines.append(f"{k}: {str(v)[:MAX_FIELD_CHARS]}") + return "\n".join(lines) or "(this record has no filled-in values)" + + +def _instruction(prompt, options, label): + return ( + f"You are deciding what happens to one record waiting at a review step called " + f"{label!r} in a workflow.\n\n" + f"The person who built this workflow told you: {prompt}\n\n" + f"Choose EXACTLY ONE of these next steps, by its exact name:\n" + + "\n".join(f"- {o}" for o in options) + + "\n\nAnswer with one line of JSON and nothing else:\n" + '{"choice": "", "reason": ""}\n' + "If the record does not give you enough to decide, answer " + '{"choice": "", "reason": "why not"} and a person will decide instead.' + ) + + +def _parse(text, options): + """The model's line → `(choice, reason)`. A choice that is not one of the offered stages is + DISCARDED, not fuzzy-matched: the offered list is a permission boundary, and a near-miss + resolved by string distance is how a card ends up somewhere nobody authorised.""" + raw = str(text or "").strip() + obj = None + m = re.search(r"\{.*\}", raw, re.S) + if m: + try: + obj = json.loads(m.group(0)) + except (ValueError, TypeError): + obj = None + if not isinstance(obj, dict): + return "", "" + choice = str(obj.get("choice") or "").strip() + reason = str(obj.get("reason") or "").strip()[:MAX_REASON] + for opt in options: + if choice.lower() == str(opt).lower(): + return str(opt), reason # the OFFERED spelling wins, never the model's + return "", reason + + +# ⭐⭐ WAVE 35 · W35-T41 / C7 — THE TWO TRANSPORTS RETURN THE RESPONSE BODY'S `usage`. +# +# ⛔ THIS FILE'S OWN HEADER USED TO BE THE PRODUCT'S CONFESSION THAT NOTHING COUNTED: `ai_enrich`'s +# docstring says *"`ai_review.decide` -- the product's only other LLM entry -- has no token +# accounting of ANY kind"*. R9 asks for ONE meter for every AI surface, so the counting has to reach +# this transport rather than be bolted onto its caller — the caller never sees the body. +# +# ⚠ A THIRD RETURN VALUE, NOT A MUTATED ARGUMENT, and both call sites are in `decide` below. The +# tuple grew from `(text, err)` to `(text, err, usage)`; `usage` is the raw `usage` OBJECT (or None), +# because reading it is `usage_ledger`'s job and a second reader here would be the two-normalizers +# shape this repo keeps paying for. +def _call_openai(p, model, system, user, timeout): + r = requests.post(p["url"], timeout=timeout, + headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}", + "Content-Type": "application/json"}, + json={"model": model, "max_tokens": 300, "temperature": 0, + "messages": [{"role": "system", "content": system}, + {"role": "user", "content": user}]}) + if r.status_code >= 400: + # ⚠ NO BODY ON A 4xx/5xx: an error envelope carries no usage, and a provider that refused + # before running the model has nothing to bill. The call is still COUNTED by `decide`. + return "", f"{p['name']} answered {r.status_code}", None + body = r.json() + choices = body.get("choices") or [] + if not choices: + return "", f"{p['name']} returned no choices", body + return str(((choices[0] or {}).get("message") or {}).get("content") or ""), "", body + + +#: What transport the last Anthropic call actually used: `'sdk'`, `'http'`, or `''` before any. +#: ⚠ PROCESS-LOCAL AND FOR REPORTING ONLY. It exists so `GET /meta` and `verify_web_agent` can say +#: WHICH path ran rather than inferring it from a requirements file nobody in this lane can edit. +LAST_ANTHROPIC_TRANSPORT = "" + + +def _call_anthropic(p, model, system, user, timeout): + """One classification turn on the Messages API, through `providers.anthropic_send`. + + ⭐ D-346 (W37-T39): this used to hand-roll the POST. It now goes through the shared wire, which + prefers the OFFICIAL `anthropic` SDK and falls back to the same raw POST when the package is + absent. The shape of what comes back is unchanged, deliberately: `anthropic_send` normalises + both transports to `(status, body)`, so every line below this call is untouched. + """ + global LAST_ANTHROPIC_TRANSPORT + import providers as _prov # noqa: PLC0415 + # ⚠ NO TOOLS ON THIS PATH. The builder omits `tool_choice` for exactly that case, so there is + # nothing to strip here — the rule lives in `anthropic_request`, beside the thing it constrains. + req = _prov.anthropic_request( + model=model, key=os.environ[p["env"]].strip(), system=system, + messages=[{"role": "user", "content": user}], tools=[], max_tokens=300) + status, body, transport = _prov.anthropic_send(req, timeout=timeout) + LAST_ANTHROPIC_TRANSPORT = transport + if status >= 400 or status == 0: + # ⭐ THE SENTENCE COMES FROM THE LADDER, NOT FROM HERE. `refusal_sentence` already names the + # vendor and the action for every status R4 enumerated, so a bare "anthropic answered 402" + # (which is what this line used to say, and what the owner quoted back at us) cannot recur. + return "", _prov.refusal_sentence("anthropic", status, json.dumps(body)[:400]), None + body = body or {} + # ⛔ stop_reason FIRST. A safety refusal is a successful 200 with an EMPTY content list, so + # reading content[0] before this check turns a refusal into an IndexError inside a run. + # ⚠ A refusal IS billed and its body carries `usage`, so the body rides back on this branch too. + if body.get("stop_reason") == "refusal": + return "", "anthropic declined to answer this record", body + parts = [b.get("text") or "" for b in (body.get("content") or []) + if isinstance(b, dict) and b.get("type") == "text"] + if not parts: + return "", "anthropic returned no text", body + return "".join(parts), "", body + + +def decide(*, prompt, options, row, fields=(), label="Review", timeout=None, st=None, user=""): + """Pick this record's next stage. Returns `(choice, meta)`. + + `choice` is "" whenever a person should decide — which is every failure mode there is. + `meta` carries `provider`, `model`, `reason` on success, and `problem` on refusal to answer. + + ⭐ W35-T41 / C7 — `st` and `user` are the USAGE LEDGER's target. Both default to absent because + this function's caller is `automation_engine.ai_decide(rt, ...)`, in another lane's fence: it + HAS the runtime and does not pass it yet, so until it does, a review's tokens are counted as + unattributed and REPORTED by `GET /usage` rather than dropped. See `usage_ledger`'s header for + why they cannot be resolved implicitly (measured: a contextvar does not survive a FastAPI + dependency). + """ + import usage_ledger # noqa: PLC0415 + opts = [str(o) for o in (options or []) if str(o).strip()] + if not opts: + return "", {"problem": "the review offers no next stages"} + if not str(prompt or "").strip(): + return "", {"problem": "the review has no prompt for the model to follow"} + live = ladder() + if not live: + return "", {"problem": "no AI provider is configured on this deployment"} + system = _instruction(prompt, opts, label) + user = "Here is the record:\n\n" + _record_text(row, fields) + tmo = float(timeout or TIMEOUT_SECONDS) + override = (os.environ.get("AIOS_AI_MODEL") or "").strip() + problems = [] + for p in live: + model = override or p["model"] + body = None + try: + text, err, body = (_call_anthropic if p["shape"] == "anthropic" else _call_openai)( + p, model, system, user, tmo) + except Exception as e: # noqa: BLE001 + text, err = "", f"{p['name']} failed: {type(e).__name__}" + # ⭐⭐ C7 — THE LEDGER LINE, BEFORE ANY BRANCH BELOW READS THE ANSWER. + # ⛔ IT IS WRITTEN ON EVERY OUTCOME THAT REACHED A PROVIDER, including a refusal and an + # unusable answer. A meter that books only successes reports a cheap week for a run that + # spent its budget being declined — and a declined call is billed. The only path that does + # NOT record is a transport that never reached the vendor (`body is None`), which spent + # nothing. + if body is not None: + ins, outs = usage_ledger.tokens_from(body) + usage_ledger.record("ai_review", p["name"], model, ins, outs, + total=usage_ledger.total_from(body), st=st, user=user) + if err: + problems.append(err) + continue # ladder: a dead provider degrades to the next one + choice, reason = _parse(text, opts) + if not choice: + # The provider ANSWERED and declined (or answered unusably). That is a decision about + # this record, not a fault in the provider, so it does NOT fall through to a more + # expensive one — the card goes to a human, which is what the model just asked for. + return "", {"provider": p["name"], "model": model, + "problem": reason or "the model did not choose one of the stages"} + return choice, {"provider": p["name"], "model": model, "reason": reason} + return "", {"problem": "; ".join(problems)[:300] or "no provider answered"} + + +# ══════════════════════════════════════════════════════════════════ the FLOW WRITER +# ⭐⭐ WAVE 33 · W33-T54/T55 (owner item 7, ruling R3) — A PROMPT BECOMES A DRAFT AUTOMATION. +# +# Owner, verbatim: *"lay out the foundation with custom tools we provide"* — and the tools are the +# action catalog, which has been machine-readable since wave 23. So this writes no new vocabulary: +# it hands the model the SAME `ACTION_CATALOG` the menu paints, the SAME `ACTION_REQUIRED` the +# runner blocks on, and the SAME trigger keys `clean_trigger` accepts, and asks for one JSON object +# in that vocabulary. +# +# ⛔ WHY IT LIVES IN `ai_review.py` RATHER THAN A NEW FILE. Two reasons, both about ownership. This +# is the one module in the API package that already holds an LLM ladder and its credentials, so a +# second one would be a second place a rotated key has to be noticed; and a new `routes_*.py` needs +# a `main.py` line from another lane (contract C2) to be reachable at all, which is how wave 23 +# shipped three finished routers 404-dead behind green gates. The DOOR is a route on +# `routes_automation.py`, which is already mounted. +# +# ⛔⛔ CONTRACT C7 IS THE DESIGN, NOT A CHECK AT THE END: *"the flow writer may emit only kinds +# present in `ACTION_CATALOG`, and its output must pass `clean_actions` unchanged."* Both halves are +# enforced mechanically rather than asked for politely — the kind list is an `enum` in the schema +# the model fills, and the caller runs `clean_actions` and DIFFS the result. That diff matters +# because `clean_actions` has no disclosure channel (D-75): it drops a key it does not recognise +# and answers 200, so a draft accepted without the diff would show a person a flow that is not the +# flow the model described, with nothing anywhere saying so. +# +# ⚠ `chat` IS INJECTABLE, exactly as `routes_query._call_model`'s is, and it is the reason this door +# can be proven end to end with no API key and no spend. A gate that can only run where a +# credential exists is a gate that never runs. + +#: Cerebras first, NOT the cheap-first order `ladder()` uses. Same reason `routes_query` inverts it: +#: this contract includes a REFUSAL, and refusing honestly ("I cannot build that from the steps you +#: have") is a model property. A cheaper rung answers an impossible request with a plausible flow, +#: which is worse than no flow. Override with `AIOS_FLOW_PROVIDER`. +FLOW_PROVIDER_ORDER = ("cerebras", "groq", "openrouter", "anthropic") +MAX_DRAFT_ACTIONS = 12 # a draft a person reads in one screen; the engine's own cap is 25 +MAX_PROMPT_CHARS = 2000 + + +def flow_providers(pin=None): + """The rungs usable here, in THIS module's refusal-first order. Empty = the feature is off. + + ⭐ `pin` IS ASK D-18 (2026-08-18): the Agent chat's model toggle must configure something, and + the draft door used to read `prompt` off the body and nothing else — so the key the client sent + was accepted and dropped, and the picker was a control over nothing. + ⚠ AN UNKNOWN OR UNCONFIGURED PIN FALLS BACK TO THE LADDER rather than refusing. A model the + ladder stopped offering must not turn every later draft into an error; the caller is told which + rung actually answered, which is the honest half. + """ + by_name = {p["name"]: p for p in PROVIDERS} + live = [n for n in FLOW_PROVIDER_ORDER + if n in by_name and (os.environ.get(by_name[n]["env"]) or "").strip()] + wanted = str(pin or os.environ.get("AIOS_FLOW_PROVIDER") or "").strip().lower() + if wanted and wanted in live: + return [by_name[wanted]] + return [by_name[n] for n in live] + + +def flow_schema(kinds, trigger_keys, table_keys): + """The tool schema the model fills — the catalog's OWN key lists as enums. + + ⛔ `kind` AND `trigger` ARE ENUMS, not free strings, and that is C7's first half enforced by the + transport rather than by a check afterwards. A model asked for "any action name" invents + `send_slack_message` on a deployment that has no such kind, and the failure then surfaces as a + 400 from `clean_actions` carrying a sentence about a word the person never typed. + ⚠ `required` IS `["kind"]` ALONE, deliberately: a REFUSAL carries no name and no actions, and a + schema demanding them turns an honest refusal into a provider-side 400 that reads exactly like a + transport failure (`routes_query._spec_schema` carries the same note, for the same measured + reason). + """ + return { + "type": "object", + "properties": { + "kind": {"type": "string", "enum": ["flow", "refused"], + "description": "refused = this cannot be built from the steps available"}, + "refusal": {"type": "string", + "description": "when kind=refused: ONE plain sentence naming what is " + "missing, in the words a non-technical person would use"}, + "name": {"type": "string", "description": "a short title for the automation"}, + "trigger": {"type": "string", "enum": sorted(trigger_keys), + "description": "what starts this automation"}, + "table": {"type": "string", "enum": sorted(table_keys), + "description": "the database whose records this flow walks, if any"}, + "actions": { + "type": "array", + "maxItems": MAX_DRAFT_ACTIONS, + "items": { + "type": "object", + "properties": { + "kind": {"type": "string", "enum": sorted(kinds)}, + "why": {"type": "string", + "description": "one short sentence: why this step is here"}, + "config": {"type": "object", "additionalProperties": True, + "description": "the step's settings, using ONLY the keys named " + "for that kind in the system message"}, + }, + "required": ["kind"], + }, + }, + }, + "required": ["kind"], + } + + +def flow_system_prompt(catalog, required, triggers, tables): + """What the model is told it may build with — DERIVED, never written down twice. + + Every list here is the server's own: the catalog rows the menu paints, the required-key table + the runner blocks on, the trigger keys `clean_trigger` accepts, and this tenant's real databases + with their real columns. Nothing about the vocabulary is restated by hand, so a kind added to + the catalog is offered here on the same deploy and a kind removed stops being offered. + """ + lines = ["You build small automations for a business tool. You are given the EXACT set of " + "steps this tool can perform. You may use nothing else.", + "", "THE STEPS YOU MAY USE:"] + # ⚠ BOTH HALVES, `key (phrase)`. The KEY is what the model must write into `config` and the + # PHRASE is the only human wording of that requirement anywhere — the one the runner's own + # refusal sentence is built from. Keys alone leave the model guessing what `field` means on a + # `web_read` (it is a column to write into, not a form field); phrases alone leave it guessing + # what to call the setting. Handing it one and hoping is how a draft comes back configured + # against a key the validator drops. + req = {str(k): [f"{key} ({phrase})" for phrase, key in v] for k, v in (required or {}).items()} + for row in catalog or []: + if not row.get("ready"): + continue + kind = str(row.get("kind") or "") + need = req.get(kind) or [] + lines.append(f"- {kind}: {row.get('label')} - {row.get('detail') or ''}" + + (f" REQUIRED settings: {', '.join(need)}" if need else "")) + lines += ["", "WHAT CAN START AN AUTOMATION:"] + for t in triggers or []: + if t.get("planned") or not t.get("ready", True): + continue + lines.append(f"- {t.get('key')}: {t.get('label')}") + lines += ["", "THE DATABASES THIS PERSON HAS:"] + for t in (tables or [])[:40]: + cols = ", ".join(str(f.get("key")) for f in (t.get("fields") or [])[:25]) + lines.append(f"- {t.get('key')} ({t.get('label')}): {cols or 'no columns yet'}") + if not tables: + lines.append("- (none - do not name a database)") + lines += [ + "", + "RULES:", + "1. Use ONLY the step kinds listed above. If what is asked needs a step that is not there, " + "answer kind=refused and say plainly which capability is missing.", + "2. Fill in every REQUIRED setting you can from what the person told you. Leave one blank " + "rather than inventing a web address, a CSS selector or a column name.", + "3. A web address must start with http:// or https://.", + "4. To use a value from the record the flow is walking, write it as {{Column name}}.", + "5. Name a database only from the list above, by its key.", + "6. Keep it short. Fewer steps that work beat more steps that guess.", + ] + return "\n".join(lines) + + +def _flow_from_tool_call(obj): + """The model's tool arguments -> `(draft, refusal)`. A shape error is a refusal, never a crash.""" + if not isinstance(obj, dict): + return None, "the assistant's answer could not be read" + if str(obj.get("kind") or "") == "refused": + return None, (str(obj.get("refusal") or "").strip() + or "the assistant could not build this from the steps available") + acts = obj.get("actions") + if not isinstance(acts, list) or not acts: + return None, ("the assistant did not produce any steps - try describing what should " + "happen, one action at a time") + out = [] + for a in acts[:MAX_DRAFT_ACTIONS]: + if not isinstance(a, dict) or not str(a.get("kind") or "").strip(): + continue + cfg = a.get("config") + out.append({"kind": str(a["kind"]).strip(), + "config": cfg if isinstance(cfg, dict) else {}, + "why": str(a.get("why") or "").strip()[:200]}) + if not out: + return None, "the assistant's steps could not be read" + return {"name": str(obj.get("name") or "").strip()[:80] or "New automation", + "trigger": str(obj.get("trigger") or "").strip(), + "table": str(obj.get("table") or "").strip(), + # ⛔⛔ THE TRUNCATION IS REPORTED, and it is a STANDING RULE that it must be (owner, + # 2026-08-12, W30/R6 second sentence): *"if there is lag or it can't be done, you need + # to explicitly tell me why and recommend a fix"* — **a silent truncation IS the + # violation, not the limit.** `acts[:MAX_DRAFT_ACTIONS]` above dropped everything past + # the twelfth and said nothing, so a model that answered with a twenty-step journey had + # eight steps deleted between the answer and the screen, with no key anywhere in the + # response naming them. `MAX_ACTIONS` is 20, so those steps were STORABLE — this + # ceiling is the draft door's own, which makes reporting it the whole obligation. + # ⚠ Found by the verifier that checked this ticket, not by a gate. + "asked": len([a for a in acts if isinstance(a, dict)]), + "actions": out}, "" + + +def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeout=None, + st=None, user="", model=None): + """A sentence -> `(draft, refusal_sentence, provider)`. ⛔ NOTHING IS SAVED HERE. + + Exactly one of `draft` and `refusal_sentence` is truthy — the same contract + `web_agent.run_step` keeps, so a caller has no third case to get wrong. + + ⭐ W35-T41 / C7 — `st`/`user` are the usage ledger's target, exactly as on `decide` above and for + the same reason: both of this function's callers (`automation_engine._ai_agent_plan` and + `routes_automation`'s draft door) are in lane D's fence. C7 says E adds the ledger line here and + D asserts it; the two keywords are what D has to pass for the line to be attributable. + """ + # ⚠ DECLARED AT THE TOP OF THE FUNCTION, not beside the assignment inside the provider loop. + # It parses either way; a reader scanning for the declaration does not look inside a `for`. + global LAST_ANTHROPIC_TRANSPORT + import usage_ledger # noqa: PLC0415 + text = str(prompt or "").strip()[:MAX_PROMPT_CHARS] + if not text: + return None, "type what you want the automation to do", None + kinds = sorted({str(r.get("kind")) for r in (catalog or []) if r.get("ready")}) + if not kinds: + return None, "this deployment offers no automation steps to build with", None + trigger_keys = sorted({str(t.get("key")) for t in (triggers or []) + if t.get("key") and not t.get("planned")}) or ["manual"] + table_keys = sorted({str(t.get("key")) for t in (tables or []) if t.get("key")}) or [""] + tools = [{"type": "function", "function": { + "name": "build_automation", + "description": "Emit the automation, or refuse.", + "parameters": flow_schema(kinds, trigger_keys, table_keys)}}] + messages = [{"role": "system", + "content": flow_system_prompt(catalog, required, triggers, tables)}, + {"role": "user", "content": text}] + + if chat is not None: + # ⚠ THE INJECTED PATH IS THE PROVEN PATH. It runs the SAME parse and the SAME refusal + # branches as a live call; only the transport is replaced. + draft, refusal = _flow_from_tool_call(chat(messages, tools)) + return draft, refusal, "injected" + + provs = flow_providers(model) + if not provs: + # ⛔ SAY SO. An AI feature that silently does nothing is indistinguishable from one that was + # never built [[flag-shipped-without-its-writer]]. + return None, ("the assistant is not configured on this deployment, so an automation cannot " + "be drafted from a description yet"), None + tmo = float(timeout or TIMEOUT_SECONDS) + problems = [] + import providers as _prov + for p in provs: + # ⭐⭐ W36-T35 / R4 — THE OWNER QUOTED THIS LINE BACK AT US. It used to read + # `problems.append(f"{p['name']}: tool calls are not wired for this shape")` and `continue`, + # so the sentence *"anthropic: tool calls are not wired for this shape"* appeared under the + # Agent module verbatim. R4: *"Anthropic becomes the tool-calling path that always works."* + # It is wired now, through the ONE wire in `providers`, and it matters more than it looks: + # every side rung on this account is refusing today (cerebras 402, groq 404, openrouter + # 402), so without this branch the drafter cannot answer at all. + # ⚠ NO `effort` ON THIS DOOR — `output_config.effort` errors on Haiku 4.5, the tier this + # ladder runs. The parameter is the caller's to send, which is why the wire takes it. + # ⭐ D-346 (W37-T39): BOTH shapes now yield the SAME `(status, body)` pair, so every branch + # below reads one thing. The anthropic side gets there through `anthropic_send` (official + # SDK when installed, the identical raw POST when not); the openai side still posts here + # because there is no shared SDK across three different vendors on that wire. + global LAST_ANTHROPIC_TRANSPORT + try: + if p["shape"] == "anthropic": + req = _prov.anthropic_request( + model=p["model"], key=os.environ[p["env"]].strip(), system=None, + messages=messages, tools=tools, max_tokens=1500, tool_choice="required") + status, body, transport = _prov.anthropic_send(req, timeout=tmo) + LAST_ANTHROPIC_TRANSPORT = transport + else: + r = requests.post(p["url"], timeout=tmo, + headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}", + "Content-Type": "application/json"}, + json={"model": p["model"], "messages": messages, "tools": tools, + "tool_choice": "required", "temperature": 0.1, + "max_tokens": 1500}) + status = r.status_code + body = r.json() if (r.content and r.status_code == 200) else {"_text": r.text} + except Exception as e: # noqa: BLE001 + problems.append(f"{p['name']}: {type(e).__name__}") + continue + if status != 200: + # ⛔ A SENTENCE, NOT `HTTP 402` (R4's third clause). The owner read the status codes off + # this very door. `refusal_sentence` also decides whether this was a CREDIT failure, and + # the memo makes the next turn SKIP the empty account instead of paying for it again. + # ⚠ The body is re-serialised for the substring test because the SDK path never had a + # `.text` — that is the one thing this normalisation costs, and it costs nothing else. + detail = json.dumps(body)[:600] if isinstance(body, dict) else str(body)[:600] + if _prov.is_credit_failure(status, detail): + _prov.mark_no_credit(p["name"]) + problems.append(_prov.refusal_sentence(p["name"], status, detail)) + continue + try: + body = body or {} + if p["shape"] == "anthropic": + _text, args, _refused = _prov.anthropic_read(body) + if _refused: + problems.append(f"{p['name']}: {_refused}") + continue + else: + calls = (((body.get("choices") or [{}])[0].get("message") or {}) + .get("tool_calls") or []) + args = json.loads(calls[0]["function"]["arguments"]) if calls else None + except Exception as e: # noqa: BLE001 + problems.append(f"{p['name']}: unreadable answer ({type(e).__name__})") + continue + # ⭐⭐ C7 — THE LEDGER LINE. Placed after the parse rather than before it so an UNREADABLE + # answer is not double-counted by the `continue` above... ⚠ which means an unreadable answer + # is NOT counted at all, and that is a deliberate, disclosed loss: a body this code cannot + # parse is a body whose `usage` it also cannot trust, and `body` is out of scope in that + # branch by construction. The 200-with-junk case is rare and named here rather than + # silently rounded to zero. + ins, outs = usage_ledger.tokens_from(body) + usage_ledger.record("automation_draft", p["name"], p["model"], ins, outs, + total=usage_ledger.total_from(body), st=st, user=user) + draft, refusal = _flow_from_tool_call(args) + if draft is None and isinstance(args, dict) and str(args.get("kind")) == "refused": + # ⛔ A REFUSAL IS AN ANSWER, NOT A FAULT, so it does NOT fall through to a more + # expensive rung — the model has just said this cannot be built. `decide()` takes the + # same posture for the same reason. + return None, refusal, p["name"] + if draft is not None: + return draft, "", p["name"] + problems.append(f"{p['name']}: {refusal}") + return None, ("; ".join(problems)[:300] or "no provider answered"), None diff --git a/api/automation_engine.py b/api/automation_engine.py index 9328b1cf37c6e5c7c73034ba9d1814d90f5b7b32..a696848fecf36f2669c5aa3d3edbe5295ae85403 100644 --- a/api/automation_engine.py +++ b/api/automation_engine.py @@ -1,14613 +1,14613 @@ -"""automation_engine.py — wave-18 item 5 (contract C4-AUTO): the automation RUNTIME. - -THE SHAPE, and why it is this shape. An automation is a DEFINITION that lives in the tenant -store and a RUN that lives in this process. The definition is durable, small and rarely written; -the run is hot, chatty and worthless five minutes later. Conflating them is the failure this file -is built to avoid, and it has two halves: - - * **Run state is NEVER persisted while it is running.** A `state: 'running'` written to the - store outlives the process that wrote it, so a container restart mid-run leaves an automation - that can never run again — the 409 sees a `running` nothing will ever clear. Hot state lives - in `_RUNNING` (a process dict, cleared by definition on restart) and the store is written - exactly ONCE per run, at the end. - * **One coalesced store update per run per bucket.** `core/store.py` coalesces on a 20 s - floor per key against a 256-commits/hour repo budget, so a per-ROW write is the wrong shape - by two orders of magnitude — the S&P demo alone is ~500 rows. Every runner below computes - its whole result first and commits it in a single `update` callback. - -WHAT IS VENDORED HERE AND WHY. `.claude/skills/browse/scrape.py` is not on `sys.path` and is not -deployed, so its extraction core is copied into this file (the wave doc's instruction), ported -httpx → requests (the only HTTP dependency this API already carries). The SSRF rail comes with -it, HARDENED rather than merely preserved — see `guard`/`fetch`. That hardening is the point: -the skill version takes a URL from a developer on a CLI; this takes one from a request body. -""" -from __future__ import annotations - -import copy -import datetime as _dt -import hashlib -import hmac -import ipaddress -import json -import os -import re -import socket -import threading -import time -from urllib.parse import urljoin, urlparse - -import requests - -# ⭐ `providers` IS DELIBERATELY NOT IMPORTED HERE ANY MORE (wave 27 item 23). The capability -# router had exactly one consumer in this file — the views top-up inside the Bright Data rung — -# and that rung now lives in `connectors_ig`. So the vendor-routing seam is reached only by the -# connector that routes, which is the shape the split was for: this file no longer knows that -# there is more than one vendor, or that vendors cost money. Re-adding this import is therefore -# a signal, not a convenience — it means something in the automation runtime started making a -# vendor decision, and that belongs one layer down. - -try: # the parser the extraction half needs - from bs4 import BeautifulSoup -except Exception: # pragma: no cover — deploy lag; see mailbox ② - BeautifulSoup = None - -# --------------------------------------------------------------------------------------------- -# THE BUCKET (C4-AUTO) -# --------------------------------------------------------------------------------------------- - -#: The per-tenant store key. Colocated with its reader, the `routes_nav._NAV_PREFS_KEY` pattern. -STORE_KEY = "automations" -#: The user-table bucket. ⚠ Read/written HERE through `TenantRuntime`, never through -#: `core.user_tables` — that module writes the UNPREFIXED module-global key, which is correct for -#: tenant #0 (empty prefix) and silently cross-tenant for every R2 tenant after it. Booked in the -#: session-D mailbox as a finding against A's file rather than fixed from here. -UT_STORE_KEY = "user_tables" -UT_PREFIX = "ut_" - -MAX_AUTOMATIONS = 40 -MAX_RUNS = 20 # trimmed history per automation -MAX_NAME = 80 -#: ⛔ IT DOES **NOT** MIRROR `core.user_tables.MAX_ROWS`, AND THE COMMENT THAT SAID SO WAS A 12x -#: STALE CLAIM — `MAX_ROWS` is 60,000. That sentence is most of what made D-143 hard to see: it -#: read as "the substrate's bound, kept in step", so nobody asked whether a flow was silently -#: walking 5,000 of 32,826 connected rows. It was. -#: **This is the FALLBACK ONLY.** The live answer is per TABLE and comes from -#: `core.user_tables.row_limit` via `_flow_record_cap` — `None` for a connected source (R6: no -#: cap), `MAX_ROWS` for the editable substrate, `0` for a read-through grid. This constant is -#: reached only when that import fails, and it is deliberately the CONSERVATIVE direction. -#: ⚠ Do not "fix" it by raising it to 60,000: a fallback that runs when the evaluator is missing -#: should not also be the widest one. And do not delete it — D-143's own row says the editable -#: substrate still needs a bound. -MAX_UT_ROWS = 5000 -MAX_UT_TABLES = 40 - -#: ⛔⛔ D-112's SENTENCE, AS A CONSTANT, BECAUSE TWO PLACES DEPEND ON IT AGREEING. -#: `enrich_selection` produces it when a step's `fromView` names a view that no longer resolves; -#: `run_flow` tests for it to turn that run `partial` instead of `ok`. Measured live on a real -#: tenant: an enrich action pointed at a deleted view walked ZERO records and reported **`ok`** — -#: indistinguishable on every surface from a run that worked, and a strong candidate for why the -#: owner's enrichment kept appearing to do nothing. -#: ⚠ THE WORDING IS FREE TO CHANGE; the AGREEMENT is not. Both sites read this name, so a better -#: sentence stays a one-line edit instead of a silent regression [[constant-two-features-share]]. -ENRICH_VIEW_UNREADABLE = "the enrich step names a view it cannot read" - -#: ⚠ THE APPEND TABLES NEED A DIFFERENT CAP, and the reason is arithmetic rather than taste. -#: `MAX_UT_ROWS` was sized for a scraped LIST — a page of ~500 companies that is re-read, so the -#: row count is bounded by the page. The `ut_ig_*` tables are the opposite shape: every pull -#: INSERTS a timestamped row (see `SNAPSHOT_FIELDS` and the compound keys below), so at R1's -#: 1k-profiles-daily the snapshot table crosses 5000 rows on **day five** — and the old behaviour -#: was a SILENT `skipped++`, i.e. the table would quietly stop growing and the run would still -#: report success. A time series that stops after five days without saying so is worse than one -#: that was never built. -#: 200k is the owner-directed ceiling (R1's "~200k"). ⚠ IT BUYS VERY DIFFERENT HORIZONS FOR THE -#: TWO APPEND TABLES, and the difference is worth knowing before anyone plans around it: -#: ut_ig_snapshots 1 row per profile per pull -> ~200 days at 1k profiles/day -#: ut_ig_post_snapshots maxPosts rows per pull -> ~8 days at 1k profiles x 24 posts -#: So the post series is the one that fills, and it fills FAST. That is exactly why the breach had -#: to become LOUD (a distinct `capped` count -> a `partial` run naming the table): at this rate the -#: silent version would have flatlined a chart inside a fortnight with a green dot over it. -#: -#: ⚠ MEASURED COST AT THE CEILING (2026-08-04, this box): a full `ut_ig_post_snapshots` serialises -#: to **35.8 MB** of JSON (~1.4 s). `UT_STORE_KEY` is ONE bucket for ALL of a tenant's user tables, -#: so that cost is paid by every unrelated automation write in the tenant too. Booked for the B-3 -#: Postgres tripwires (R2 clause b) rather than papered over — the fix is a row store, not a -#: smaller number. -#: (W19-C; the GENERIC loud-breach for every other table stays booked as DEBT D-11.) -MAX_UT_IG_ROWS = 200_000 -IG_TABLE_PREFIX = "ut_ig_" - -#: ⚠ THE RAISED CEILING BELONGS TO THE **APPEND** TABLES, NOT TO A NAME PREFIX (wave 20). It was -#: keyed off `ut_ig_`, which was exactly right while every `ut_ig_*` table was an append table — -#: and stopped being right the moment discovery added `ut_ig_candidates`, which is UPSERTED BY -#: HANDLE and is bounded by how many accounts exist rather than by how often we look. It would -#: have inherited a 200,000-row ceiling, and with it the measured 35.8 MB single-bucket -#: serialisation cost, purely by an accident of naming. Naming the append tables is the version -#: that stays true when the next `ut_ig_*` table is not one. -IG_SNAPSHOTS_TABLE = "ut_ig_snapshots" -IG_POSTS_TABLE = "ut_ig_posts" -IG_POST_SNAPSHOTS_TABLE = "ut_ig_post_snapshots" -IG_COMMENTS_TABLE = "ut_ig_comments" - -# --------------------------------------------------------------------------------------------- -# ⭐⭐ WAVE 29 (item 7, DEBT D-9, owner rulings R1 + R2) — TIKTOK, AS A PARALLEL FAMILY -# --------------------------------------------------------------------------------------------- -# R1: FULL PARITY — profile AND posts AND comments, not a profile tier. -# R2: a PARALLEL `ut_tt_*` family. `ut_ig_*` is untouched: zero migration, zero risk to the live -# Instagram rows, and the two schemas may DIVERGE where the vendors do. -# -# ⛔ WHY A SECOND FAMILY RATHER THAN A `platform` COLUMN ON THE FIRST, stated here because it is -# the question every reader will ask. `PRESET_PROFILE_FIELDS` carries `platform` and its identity -# is `(platform, handle)` — so a TikTok PROFILE row could already have lived in `ut_ig_profile`. -# The other four tables carry NO discriminator at all: `ut_ig_posts`/`ut_ig_comments` key on -# `shortcode` and the snapshot tables on `influencer_key`, so an Instagram post and a TikTok video -# that happened to share a code would MERGE SILENTLY. Adding `platform` to four live append tables -# holding hundreds of thousands of rows is a migration on production data to buy a shared grid -# nobody asked for. R2 chose the version with no migration. -# -# ⚠ THE ACCEPTED COST, so it is not rediscovered as a defect: the engine, rollups and grids learn -# two families, and a cross-platform view needs a union. In exchange the two schemas can be HONEST -# about their vendors — which is why `ut_tt_posts` has no `plays` column and `ut_tt_profile` says -# `tt_id` rather than inheriting a column labelled "Instagram id". -TT_TABLE_PREFIX = "ut_tt_" -TT_PROFILE_TABLE = "ut_tt_profile" -TT_SNAPSHOTS_TABLE = "ut_tt_snapshots" -TT_POSTS_TABLE = "ut_tt_posts" -TT_POST_SNAPSHOTS_TABLE = "ut_tt_post_snapshots" -TT_COMMENTS_TABLE = "ut_tt_comments" - -AUTOMATION_RECORD_MODE = "automation" -#: ⚠ THE RAISED CEILING FOLLOWS THE APPEND SHAPE, NOT THE PLATFORM. `ut_tt_snapshots` and -#: `ut_tt_post_snapshots` are append tables for exactly the reason their IG twins are — one row per -#: profile per pull, `maxPosts` rows per pull — so they inherit the ceiling by JOINING THIS SET, -#: which is the mechanism the note above says survives the next table that is not an append table. -APPEND_TABLES = frozenset({IG_SNAPSHOTS_TABLE, IG_POST_SNAPSHOTS_TABLE, - TT_SNAPSHOTS_TABLE, TT_POST_SNAPSHOTS_TABLE}) - -#: ⭐ WAVE 24 (owner ruling R6) — `plain` IS WHAT AN AUTOMATION IS NOW, and it is the DEFAULT. -#: The create wizard is deleted, so nobody picks a kind any more: a new automation is a trigger -#: plus actions and NO machine step. The other three are MACHINE kinds — a scrape, an Instagram -#: column, an Instagram search — and they survive on the automations that already use them -#: (`discover_instagram` stays reachable through the `ig_profile_match` trigger; see C-TRIG). -#: -#: ⚠ ADDING A KIND HERE IS THE SMALL HALF, and the reason is worth reading before adding a fifth: -#: TWO dispatches in this file used to end in a bare `else` that belonged to a SPECIFIC kind -#: rather than to a default — `graph()`'s was `scrape_db`'s and `compose_sentence`'s was -#: `discover_instagram`'s. A kind added here alone would have inherited another kind's whole -#: description: three machine nodes it does not have, and a one-line summary about searching -#: Instagram for up to 0 profiles. Both are explicit arms now, and neither has an `else`. -#: ⭐ WAVE 29 (D-9 / R1) — `discover_tiktok` joins, reachable ONLY through the -#: `tiktok_profile_match` trigger (the same law that makes `discover_instagram` reachable). It -#: lands here IN THE SAME CHANGE as its `RUNNERS` entry and its flip law: a kind in this tuple -#: with no runner is a control that must refuse. -KINDS = ("plain", "scrape_db", "field_instagram", "discover_instagram", "discover_tiktok") -#: ⭐⭐ WAVE 30 · T04 — THE DISCOVERY KINDS UNDER ONE NAME, and this constant is a bug fix rather -#: than tidying. `discover_tiktok` shipped in wave 29 by being added to `KINDS`, `RUNNERS`, -#: `clean_config` and `TRIGGER_*` — four sites that were found — while FIVE more tested the string -#: `"discover_instagram"` directly and were not. The visible symptom was the owner's: picking the -#: TikTok trigger 400'd with *"say how many profiles to fetch"* on the FIRST save, because the seed -#: below was one of the five. The other four are silent — a canvas with no `find` panel, a flow -#: table resolving to the wrong default, and a summary sentence announcing the automation would -#: "do nothing yet". -#: -#: ⛔ SO THE RULE IS: A DISCOVERY BRANCH TESTS MEMBERSHIP OF THIS TUPLE, NEVER A KIND STRING. -#: Three parallel string comparisons is precisely how the third platform gets missed twice more, -#: and the misses are individually invisible — each one degrades a different surface, none of them -#: raises, and every gate stays green (this whole family shipped green in wave 29). -#: ⚠ The kind ↔ platform facts that genuinely DIFFER — the dataset id, the default target table, -#: the field map — stay resolved per kind where they are used. This tuple answers *"is this a -#: corpus search?"* and nothing else; widening it into a platform registry would just move the -#: problem somewhere with a longer name. -DISCOVERY_KINDS = ("discover_instagram", "discover_tiktok") -#: ⭐ WAVE 30 · T06 — THE TRIGGER HALF, and it is a SEPARATE map because the two are NOT -#: interchangeable, which cost a red to learn. Law 1 makes a discovery TRIGGER choose its kind, so -#: `trigger ⇒ kind` always holds — but the converse does NOT: a definition can carry -#: `kind: "discover_instagram"` with no trigger at all (a direct API create does exactly that, and -#: `verify_automation`'s `_discover_defn` fixture is one). Testing the KIND where the rule is about -#: the trigger therefore fires on definitions the picker could never produce — measured: it spawned -#: a preset database under a dry-run fixture that asserts none exists. -#: ⛔ SO: "did somebody PICK a corpus search in the picker?" reads THIS. "Is this stored definition a -#: corpus search?" reads `DISCOVERY_KINDS`. Two questions, two maps, and the gate asserts this one -#: agrees with law 1 rather than trusting that it does. -DISCOVERY_TRIGGER_KIND = {"ig_profile_match": "discover_instagram", - "tiktok_profile_match": "discover_tiktok"} -#: ⭐ WAVE 30 · T08 — the ENRICH action kinds, one per network. Same argument as `DISCOVERY_KINDS` -#: one paragraph up: these two share a validator, a selection, a cooldown and a run summary, and the -#: only things that differ are which connector answers and which tables the rows land in. -#: ⛔ ONE VALIDATOR, NOT TWO. `clean_actions`' enrich branch is ~40 lines of clamps whose comments -#: record why each one is shaped the way it is (`submitted=False` because the client re-posts the -#: whole action list; `limit` clamped twice because the run sees STORED configs). A second copy for -#: TikTok would start identical and drift, and the way it fails is that one network silently accepts -#: a limit the other refuses. -ENRICH_KINDS = ("enrich_instagram", "enrich_tiktok") -#: What a definition with no kind becomes (C-TRIG law 2) — the shape `POST /automations` stores -#: when the body names none, which after R6 is every create the client makes. -DEFAULT_KIND = "plain" -#: R6: no NEW automation may be either of these. They still RUN, still PATCH and still validate — -#: the ruling retires the door, not the two automations behind it (see `create`). -#: ⚠ `discover_instagram` is NOT here: it stays creatable, through the `ig_profile_match` trigger. -RETIRED_KINDS = ("scrape_db", "field_instagram") -#: ⛔ DEBT D-65 — WHAT TO DO INSTEAD, said at every door that refuses one of these. The kinds were -#: not deleted, they were REPLACED by actions any flow can take, and a refusal that does not name -#: the replacement sends somebody looking for a bug in a decision made on purpose. One sentence -#: per kind, in one place, because `create` and `clean_definition` both say it. -RETIRED_KIND_REPLACEMENT = { - "field_instagram": "Add the 'Enrich Instagram profile' action to any automation instead", - "scrape_db": "Add a scrape step to any automation instead", -} -#: ⛔ D-65 — THE RETIRED KINDS THAT ACTUALLY HAVE SOMEWHERE ELSE TO GO, and the distinction is the -#: whole reason this is a second, narrower tuple rather than a reuse of `RETIRED_KINDS`. -#: `field_instagram` was REPLACED: W25/R4 shipped `enrich_instagram` as a `ready:true` action any -#: flow can take, so refusing the kind costs a person nothing but a different click. -#: ⚠ `scrape_db` IS DELIBERATELY ABSENT. Its replacement — the five `web_*` actions — is declared -#: `ready:false` and is DEBT D-51, so refusing a PATCH to it would delete the only way to build a -#: scrape automation and call it tidying up. W24/R6 retired its CREATE door and kept the patch -#: door open on purpose, and there is a gate check that says so in those words. A kind may only be -#: walled off at a door once something else answers the same need. -#: (`REPLACED_KINDS` was here and is deleted with the refusal it gated — see `clean_definition`.) -#: ⭐ WAVE 34 · R12 — `plain` READS "Agent". The owner renamed the module to Agents, so the KIND -#: that means "an ordinary one of these" is an Agent, singular (the module is the plural). -#: ⛔ THE KEY `"plain"` IS UNTOUCHED and so is every other key here: D-65's rule, restated three -#: times in this file, is that a kind is permanent — the LABELS are display and may move freely, -#: the keys are stored on every automation ever created and may not. -KIND_LABELS = {"plain": "Agent", "scrape_db": "Web page to database", - "field_instagram": "Instagram profile column", - "discover_instagram": "Find Instagram profiles", - "discover_tiktok": "Find TikTok profiles"} -#: ⭐⭐ WAVE 34 · CONTRACTS C3 + C4 — WHY A SYSTEM AGENT CANNOT BE DELETED, one sentence per slug. -#: -#: ⛔ ONE PLACE, because two doors ask this question (the DELETE route refuses with it, and the -#: Canvas explains why its controls are read-only) and a refusal that says something different -#: from the surface it refuses on is worse than either sentence alone. D-65's own remedy in a new -#: family: a refusal that does not name the alternative sends somebody looking for a bug in a -#: decision made on purpose. -SYSTEM_AGENT_REASON = { - "field_agent": "this agent is an AI enrichment column. Delete the column on its database and " - "the agent goes with it", - "odoo_sync": "this is the Odoo connection's own sync schedule. Disconnect Odoo in Connectors " - "to stop it, or change how often it runs on this page", -} - - -def system_agent_refusal(slug): - """The sentence for a `system:` slug, or a general one for a slug nobody has written yet. - - ⚠ IT NEVER RETURNS EMPTY. A refusal with no sentence is a 409 a person cannot act on, and the - fallback is the one branch that will be reached by a slug added later and forgotten here. - """ - return SYSTEM_AGENT_REASON.get(str(slug or ""), "this agent is part of the workspace setup " - "and cannot be deleted here") - - -EXTRACTS = ("table", "jsonld") -STATES = ("idle", "running", "ok", "error", "partial") -#: Which capture rung a `field_instagram` automation is allowed to reach for (R1's hybrid). -#: `anonymous` = the $0 ladder only. `brightdata` = the paid rung FIRST, then the ladder as a -#: fallback (unless the fallback is switched off — see `graph`'s `fallback` toggle). -TIERS = ("anonymous", "brightdata") -#: ⚠ STORED CONFIGS SAY `hiker`, AND THEY MEAN "THE PAID RUNG" (wave-20 D-21). The vendor swap -#: must not silently answer that request with the free ladder: `clean_config` refuses an unknown -#: tier by falling back to `anonymous`, so without this alias every existing Instagram automation -#: would quietly stop reaching for exact counts and nothing would say so. Mapping FORWARD keeps -#: the user's expressed intent (they turned the paid step ON) at a cost of ~$0.0015 a profile. -TIER_ALIASES = {"hiker": "brightdata"} - - -def clean_tier(raw): - """A stored/posted tier → a tier this engine runs, or '' when it is neither.""" - t = str(raw or "").strip().lower() - t = TIER_ALIASES.get(t, t) - return t if t in TIERS else "" - - -def row_cap(table_key): - """The row ceiling for ONE table. Per-table rather than global — see `MAX_UT_IG_ROWS`. - - An APPEND table (one row per subject per pull, forever) gets the raised ceiling; everything - else — including the `ut_ig_` table that is an UPSERT — keeps the list-table one. - """ - return MAX_UT_IG_ROWS if str(table_key or "") in APPEND_TABLES else MAX_UT_ROWS - -#: Schedule presets the editor offers. Kept here (not in the client) so the vocabulary the UI -#: shows and the vocabulary the parser accepts cannot drift. -CRON_PRESETS = [ - {"cron": "*/15 * * * *", "label": "Every 15 minutes"}, - {"cron": "0 * * * *", "label": "Hourly"}, - {"cron": "0 6 * * *", "label": "Daily at 06:00"}, - {"cron": "0 6 * * 1", "label": "Weekly (Monday 06:00)"}, - {"cron": "0 6 1 * *", "label": "Monthly (1st, 06:00)"}, -] - -UA = "Mozilla/5.0 (compatible; AIOS-automation/1.0; +https://aios.local/automation)" - - -def _now(): - return _dt.datetime.now() - - -def _stamp(dt=None): - return (dt or _now()).strftime("%Y-%m-%d %H:%M") - - -def _iso(dt=None): - """An ISO stamp **WITH its UTC offset** — `2026-08-05T14:03:11+07:00` (DEBT D-18). - - ⚠ THE OFFSET IS NOT COSMETIC. These stamps are the time axis of the `ut_ig_*` append tables - and they are rendered by a browser, which can only subtract from an instant it can LOCATE. A - naive `2026-08-05T14:03:11` is read as the *reader's* local time, so a container running UTC - minted cells a Jakarta browser would place seven hours in the future — and "2 minutes ago" is - not expressible at all. With the offset the same string is an instant, and relative rendering - becomes possible without migrating a single stored row. - - ⚠ `_parse_iso` reads it BACK as naive local ON PURPOSE. Every cron / `is_due` comparison in - this module is against a naive `_now()`, and mixing aware and naive datetimes raises - `TypeError` — so the offset rides on the WIRE and never enters the arithmetic. - """ - dt = dt or _now() - if dt.tzinfo is None: - dt = dt.astimezone() # a naive stamp from this process IS local time - return dt.isoformat(timespec="seconds") - - -#: ⭐ WAVE 26 · R3 — the DAY out of any stamp we have ever written, for the `date`-typed columns. -#: -#: ⛔ IT MUST READ EVERY SHAPE THE STORE HOLDS, which is the same trap `_parse_iso` documents one -#: function down: `first_found` cells exist as post-D-18 offset stamps -#: (`2026-08-05T14:03:11+07:00`), as pre-D-18 naive ones (`2026-08-05T14:03:11`), as `_stamp()`'s -#: space-separated minute form (`2026-08-05 14:03`) and, after this wave, as bare days. A -#: converter that understood only the newest shape would blank the oldest rows — and a migration -#: that empties cells is indistinguishable from one that moved them. -#: ⚠ Returns "" for anything it cannot read rather than guessing a day. The migration treats "" -#: as LEAVE ALONE, never as a value to write, so an unparseable cell keeps its original text and -#: shows up as itself instead of disappearing. -def _day(s): - """`2026-08-05T14:03:11+07:00` → `2026-08-05`. "" when there is no day in there.""" - raw = str(s or "").strip() - if not raw: - return "" - head = raw.replace("T", " ").split(" ")[0] - try: - _dt.date.fromisoformat(head) - except ValueError: - dt = _parse_iso(raw) - return dt.strftime("%Y-%m-%d") if dt else "" - return head - - -#: ⭐ WAVE 26 · AMENDMENT C1-a — the vendor's 0–1 engagement fraction → this product's 0–100 `pct`. -#: -#: MEASURED on corpus rows: `0.0074`, `0.0656`, `0.0014`, `0.0148`, `0.0274`, `0.0091`. Our `pct` -#: renderer prints the stored number and appends `%`, so storing the raw fraction would show every -#: creator in the book as `0.0%` — a measurement replaced by a wrong measurement, which is worse -#: than the blank cell it came from ([[analyst-chart-library]]: this repo already carries a 0–1 -#: dialect and a 0–100 dialect, and they meet here). -#: ⚠ BLANK STAYS BLANK. `""` means the vendor did not send one — both scrape probe rows were null -#: — and `0.0` would claim we measured zero engagement. -def _pct100(v): - """A 0–1 engagement fraction → a 0–100 percentage. "" when there is nothing to convert.""" - if v is None or (isinstance(v, str) and not v.strip()): - return "" - try: - return _s(round(float(v) * 100.0, 4)) - except (TypeError, ValueError): - return "" - - -def _parse_iso(s): - """A stamp → a NAIVE LOCAL datetime, with or without an offset. - - Both shapes exist in the store simultaneously and always will: every run committed before - D-18 wrote a naive stamp, and nothing rewrites history. A parser that understood only the new - shape would silently return None for them — and `is_due` reads None as "never ran", which - would re-fire every schedule once. Reading both is what makes the change additive. - """ - raw = str(s or "").strip().replace("Z", "+00:00") - dt = None - try: - dt = _dt.datetime.fromisoformat(raw) - except Exception: # noqa: BLE001 - try: - dt = _dt.datetime.strptime(raw[:19], "%Y-%m-%dT%H:%M:%S") - except Exception: # noqa: BLE001 - return None - return dt.astimezone().replace(tzinfo=None) if dt.tzinfo is not None else dt - - -# --------------------------------------------------------------------------------------------- -# THE SSRF RAIL — vendored from scrape.py `guard`, then hardened for a SERVER-SIDE fetcher -# --------------------------------------------------------------------------------------------- -# The skill version guards the URL a developer typed. This one guards a URL that arrived in a -# request body, which changes the threat model in two ways the original does not cover: -# -# 1. REDIRECTS. `httpx.Client(follow_redirects=True)` / `requests.get(allow_redirects=True)` -# never re-enter the guard, so `http://evil.example/x` → 302 → `http://169.254.169.254/…` -# sails straight past a guard that only ever saw the first URL. The negative control ("a -# localhost URL is refused") would still pass while the rail was wide open. `fetch` below -# therefore takes the hops MANUALLY and re-guards every one. -# 2. DNS. A perfectly public hostname may resolve to a private address. `guard` resolves the -# host and checks EVERY answer, not just the literal. -# -# ⚠ STATED, NOT HIDDEN: this is check-then-connect, so a DNS-rebinding attacker who flips the -# record between the guard and the socket is not stopped by it. Closing that needs a pinned -# connection to the validated IP (a custom adapter). Out of scope for v1 and recorded here rather -# than implied away — the rail refuses the realistic cases and says what it does not cover. - -class Refused(ValueError): - """The rail refused a URL. A distinct type so a refusal is never logged as a fetch error.""" - - -def _ip_public(ip): - return not (ip.is_private or ip.is_loopback or ip.is_link_local - or ip.is_reserved or ip.is_unspecified or ip.is_multicast) - - -def guard(url, allowed=()): - """PUBLIC-WEB-ONLY rail: http(s) only, no loopback/private/link-local host, DNS answers - checked too, plus the optional `--allowed` domain rail scrape.py carries.""" - p = urlparse(url or "") - if p.scheme not in ("http", "https"): - raise Refused(f"only http(s) allowed, not {p.scheme!r}") - host = (p.hostname or "").strip() - if not host: - raise Refused("no host in the URL") - low = host.lower() - if (low in ("localhost", "localhost.localdomain") - or low.endswith(".local") or low.endswith(".internal") - or low.endswith(".localhost")): - raise Refused(f"refused non-public host {host!r} (public web only)") - try: # a bare-IP host is decided on the literal - ip = ipaddress.ip_address(low) - except ValueError: - ip = None - if ip is not None and not _ip_public(ip): - raise Refused(f"refused non-public IP {host} (public web only)") - if allowed and not any(low == d.lower() or low.endswith("." + d.lower()) for d in allowed): - raise Refused(f"host {host!r} not in the allowed domains {list(allowed)}") - if ip is None: - try: - infos = socket.getaddrinfo(host, None) - except Exception as e: - raise Refused(f"could not resolve {host!r}: {type(e).__name__}") - for info in infos: - addr = info[4][0] - try: - resolved = ipaddress.ip_address(addr.split("%")[0]) - except ValueError: - continue - if not _ip_public(resolved): - raise Refused( - f"refused {host!r}: it resolves to the non-public address {resolved}") - return True - - -def fetch(url, timeout=20.0, max_kb=2048, allowed=(), max_hops=5, headers=None): - """GET `url`, taking redirects BY HAND so every hop passes `guard`. - - Returns `(status, final_url, body_bytes)`. A hop chain longer than `max_hops` is a refusal, - not a silent truncation — an endless redirect is indistinguishable from an attempt to walk - the fetcher somewhere it was told not to go. - """ - current = url - hdrs = {"User-Agent": UA, "Accept-Language": "en-US,en;q=0.9"} - hdrs.update(headers or {}) - for _hop in range(max_hops + 1): - guard(current, allowed) - r = requests.get(current, timeout=timeout, headers=hdrs, - allow_redirects=False, stream=True) - if r.status_code in (301, 302, 303, 307, 308): - loc = r.headers.get("Location") - r.close() - if not loc: - raise Refused(f"redirect {r.status_code} with no Location header") - current = urljoin(current, loc) - continue - body = r.raw.read(max_kb * 1024, decode_content=True) or b"" - status, final = r.status_code, str(r.url) - r.close() - return status, final, body - raise Refused(f"more than {max_hops} redirects. Refusing to follow further") - - -def fetch_json(url, body, timeout=60.0, max_kb=8192, headers=None): - """POST a JSON body through the SAME guard. Returns `(status, body_bytes)`. - - ⛔ NO REDIRECT FOLLOWING, AND THAT IS THE WHOLE DIFFERENCE FROM `fetch`. `fetch` walks hops by - hand because a redirected GET is ordinary. A redirected POST is not: re-sending a - credential-bearing body to a Location the *server* chose is precisely the hop the SSRF rail - exists to refuse, and `requests`' own `allow_redirects=True` would do it without ever - re-entering `guard`. So a 3xx here is a REFUSAL with the reason on it, never a second request. - (The vendor calls below are the only POSTs this module makes, and they carry the API key.) - """ - guard(url) - hdrs = {"User-Agent": UA, "Content-Type": "application/json", "Accept": "application/json"} - hdrs.update(headers or {}) - r = requests.post(url, timeout=timeout, headers=hdrs, allow_redirects=False, - data=json.dumps(body if body is not None else {}), stream=True) - try: - if r.status_code in (301, 302, 303, 307, 308): - raise Refused(f"the POST answered {r.status_code}. A redirected POST is refused, " - f"never re-sent to a location the server picked") - return r.status_code, (r.raw.read(max_kb * 1024, decode_content=True) or b"") - finally: - r.close() - - -# --------------------------------------------------------------------------------------------- -# EXTRACTION — vendored from scrape.py, unchanged in behaviour -# --------------------------------------------------------------------------------------------- - -def _soup(body): - if BeautifulSoup is None: - raise RuntimeError( - "beautifulsoup4 is not installed in this environment. The automation engine's " - "extraction half needs it (see the wave-18 mailbox: add beautifulsoup4 + lxml to " - "aios-web/requirements.txt).") - try: - return BeautifulSoup(body, "lxml") - except Exception: - return BeautifulSoup(body, "html.parser") - - -def tables(soup): - """Every HTML table as rows. Dict rows when the first row looks like a header.""" - out = [] - for t in soup.find_all("table"): - rows = [] - for tr in t.find_all("tr"): - cells = [c.get_text(" ", strip=True) for c in tr.find_all(["th", "td"])] - if cells: - rows.append(cells) - if not rows: - continue - head, body = rows[0], rows[1:] - if body and len(head) == len(body[0]) and all(head): - out.append([dict(zip(head, r)) for r in body if len(r) == len(head)]) - else: - out.append(rows) - return out - - -def jsonld(soup): - out = [] - for s in soup.find_all("script", attrs={"type": "application/ld+json"}): - try: - out.append(json.loads(s.string or s.get_text())) - except Exception: - pass - return out - - -def meta(soup): - m = {} - if soup.title and soup.title.string: - m["title"] = soup.title.string.strip() - for tag in soup.find_all("meta"): - k = tag.get("name") or tag.get("property") - v = tag.get("content") - if k and v and (k in ("description", "keywords", "author") - or k.startswith(("og:", "twitter:"))): - m[k] = v.strip() - return m - - -def preview(url, extract="table", table_index=0, allowed=()): - """The field-map preview the editor calls BEFORE anything is created: what columns does this - page actually offer, and what do the first rows look like? Never writes.""" - status, final, body = fetch(url, allowed=allowed) - if not (200 <= status < 300): - return {"ok": False, "status": status, "url": final, - "note": f"the page answered {status}. It may be bot-gated or gone", - "columns": [], "sample": [], "rowCount": 0} - soup = _soup(body) - rows = [] - if extract == "jsonld": - blocks = jsonld(soup) - flat = [] - for b in blocks: - if isinstance(b, list): - flat.extend([x for x in b if isinstance(x, dict)]) - elif isinstance(b, dict): - items = b.get("itemListElement") - flat.extend([x for x in items if isinstance(x, dict)] if isinstance(items, list) - else [b]) - rows = [{k: _scalar(v) for k, v in d.items()} for d in flat] - else: - found = tables(soup) - idx = max(0, min(int(table_index or 0), len(found) - 1)) if found else 0 - picked = found[idx] if found else [] - rows = [r for r in picked if isinstance(r, dict)] - cols, seen = [], set() - for r in rows[:50]: - for k in r: - if k not in seen: - seen.add(k) - cols.append(k) - return {"ok": True, "status": status, "url": final, "columns": cols, - "sample": rows[:8], "rowCount": len(rows), - "tableCount": len(tables(soup)) if extract != "jsonld" else 0, - "title": (meta(soup) or {}).get("title", "")} - - -def _scalar(v): - if isinstance(v, (str, int, float)) and not isinstance(v, bool): - return str(v) - if isinstance(v, bool): - return "1" if v else "" - if isinstance(v, dict): - return str(v.get("name") or v.get("@id") or "") - if isinstance(v, list): - return ", ".join(_scalar(x) for x in v[:8]) - return "" - - -# --------------------------------------------------------------------------------------------- -# CRON — a 5-field parser and a "what was the last fire time" walk -# --------------------------------------------------------------------------------------------- -# Deliberately NOT `croniter` (a dependency for ~60 lines) and deliberately NOT a forward -# scheduler. The question a tick asks is backwards-looking — *"was there a scheduled minute -# between the last run and now?"* — and answering it that way is what makes a missed tick -# self-healing: a container that was asleep for two hours runs once on wake, not eleven times and -# not never. - -_FIELD_RANGES = [(0, 59), (0, 23), (1, 31), (1, 12), (0, 6)] - - -def _parse_field(spec, lo, hi): - out = set() - for part in str(spec).split(","): - part = part.strip() - if not part: - raise ValueError("empty cron field part") - step = 1 - if "/" in part: - part, _, s = part.partition("/") - step = int(s) - if step < 1: - raise ValueError("cron step must be >= 1") - if part in ("*", "?"): - a, b = lo, hi - elif "-" in part.lstrip("-"): - a_s, _, b_s = part.partition("-") - a, b = int(a_s), int(b_s) - else: - a = b = int(part) - if a < lo or b > hi or a > b: - raise ValueError(f"cron value {part!r} out of range {lo}..{hi}") - out.update(range(a, b + 1, step)) - return out - - -def parse_cron(expr): - """`'m h dom mon dow'` → `(minutes, hours, doms, months, dows, dom_restricted, dow_restricted)`. - - Raises ValueError on anything malformed — a schedule that cannot be parsed must be refused at - WRITE time, because a cron string nobody can evaluate is an automation that silently never - runs and reports no error anywhere. - """ - parts = str(expr or "").split() - if len(parts) != 5: - raise ValueError("a cron schedule has exactly 5 fields: minute hour day month weekday") - sets = [_parse_field(p, lo, hi) for p, (lo, hi) in zip(parts, _FIELD_RANGES)] - dom_r = parts[2].strip() not in ("*", "?") - dow_r = parts[4].strip() not in ("*", "?") - return (sets[0], sets[1], sets[2], sets[3], sets[4] | ({0} if 7 in sets[4] else set()), - dom_r, dow_r) - - -def _day_matches(day, doms, months, dows, dom_r, dow_r): - if day.month not in months: - return False - # POSIX rule: with BOTH day-of-month and weekday restricted the match is the UNION, not the - # intersection. `0 6 1 * 1` means "the 1st, and every Monday" — getting this backwards makes - # a schedule that looks right fire almost never. - dow = (day.weekday() + 1) % 7 # python Mon=0 -> cron Sun=0 - if dom_r and dow_r: - return day.day in doms or dow in dows - if dom_r: - return day.day in doms - if dow_r: - return dow in dows - return True - - -def prev_fire(expr, now=None, lookback_days=400): - """The most recent scheduled minute at or before `now`, or None inside the lookback. - - Walks by DAY (≤400 iterations) rather than by minute (≥500k) — the day fields decide first, - and only a matching day needs its hours/minutes searched. - """ - minutes, hours, doms, months, dows, dom_r, dow_r = parse_cron(expr) - now = (now or _now()).replace(second=0, microsecond=0) - for back in range(lookback_days + 1): - day = (now - _dt.timedelta(days=back)).date() - if not _day_matches(day, doms, months, dows, dom_r, dow_r): - continue - same_day = back == 0 - for h in sorted(hours, reverse=True): - if same_day and h > now.hour: - continue - for m in sorted(minutes, reverse=True): - if same_day and h == now.hour and m > now.minute: - continue - return _dt.datetime(day.year, day.month, day.day, h, m) - return None - - -def is_due(defn, now=None): - """Should the scheduler run this automation right now? - - Due when a scheduled minute exists strictly AFTER the reference point (the last run, else the - moment the schedule was enabled, else creation) and at or before now. Anchoring on - `enabledAt` rather than firing on the first tick is what stops "enable a daily 06:00 job at - 14:00" from running immediately and looking like a bug. - """ - if not isinstance(defn, dict): - return False - sched = defn.get("schedule") or {} - if not sched.get("enabled"): - return False - try: - fire = prev_fire(sched.get("cron"), now=now) - except ValueError: - return False - if fire is None: - return False - since = (_parse_iso((defn.get("status") or {}).get("lastRunAt")) - or _parse_iso(sched.get("enabledAt")) - or _parse_iso(defn.get("created"))) - return since is None or fire > since - - -def next_fire(expr, now=None, lookahead_days=400): - """The next scheduled minute strictly after `now` — display only (the rail is `is_due`).""" - try: - minutes, hours, doms, months, dows, dom_r, dow_r = parse_cron(expr) - except ValueError: - return None - now = (now or _now()).replace(second=0, microsecond=0) - for ahead in range(lookahead_days + 1): - day = (now + _dt.timedelta(days=ahead)).date() - if not _day_matches(day, doms, months, dows, dom_r, dow_r): - continue - same_day = ahead == 0 - for h in sorted(hours): - if same_day and h < now.hour: - continue - for m in sorted(minutes): - if same_day and h == now.hour and m <= now.minute: - continue - return _dt.datetime(day.year, day.month, day.day, h, m) - return None - - -# --------------------------------------------------------------------------------------------- -# THE UPSERT — pure, so the arithmetic is testable without a store or a network -# --------------------------------------------------------------------------------------------- - -#: ⛔⛔ D-116 — THE SIX CELLS A CORPUS RE-FIND MUST NOT OVERWRITE ONCE AN ENRICHMENT MEASURED THEM. -#: A discovery re-find emits these on EVERY match, not only on insert, so the moment a scheduled -#: search re-matched an account somebody had PAID to enrich, six exact measurements were replaced by -#: pre-collected corpus values of unknown vintage — no error, no visible change other than the -#: number. Measured 2026-08-10 on a live tenant: 0 of 105 profiles had been re-found yet, so this -#: was a defect waiting for its first scheduled re-run rather than one anybody had seen. -#: ⚠ SCALARS ONLY, and that is the ruling. The OBSERVATION is appended either way — the corpus -#: genuinely saw that account at that follower count, and the snapshot series is where a corpus read -#: belongs. Suppressing the observation would be the same defect arriving from the other side. -CORPUS_SOFT_KEYS = ("followers", "following", "avg_engagement", "verified", "category", "bio") - - -def corpus_protect(before, _src): - """Which keys this incoming CORPUS row may not overwrite on `before`. D-116's precedence rule. - - ⛔ THE TEST IS `enriched_at`, i.e. "did an exact read ever write this row", NOT "is the cell - non-empty". The exit condition rules the second one out by name, and rightly: *"NOT by making - `upsert_rows` skip non-empty cells — that would break every re-scrape in the product."* A - re-scrape SHOULD move a number the corpus owns; what it may not do is move one an exact read - owns. - """ - return CORPUS_SOFT_KEYS if str((before or {}).get("enriched_at") or "").strip() else () - - -def upsert_rows(existing, incoming, key_field, cap=None, protect=None): - """Merge scraped rows into a user table's rows BY KEY. Returns `(rows, counts)`. - - `protect` is an optional `f(before, src) -> keys` naming, PER ROW, the keys this incoming row - may not overwrite. Default `None` = the old behaviour exactly, so the other nine call sites are - untouched — the precedence rule belongs to the CALLER that knows its data's provenance, not to - the merge, and a rule baked in here would apply to nine paths that never asked for one. - - THE RULE THAT MATTERS: **an orphan is COUNTED, NEVER DELETED.** A row that has stopped - appearing on the source page has not necessarily stopped existing — the page changed its - filter, the fetch was partial, the site paginated. Deleting on absence turns any upstream - hiccup into silent data loss, and the row may be carrying user-typed overlay values in - columns the scrape never touches. So the run reports `orphans: N` and leaves them alone. - - Only MAPPED keys are written: a re-run never clobbers a column a user added by hand. - - ⚠ CALL THIS ONCE PER TABLE PER RUN, NOT ONCE PER ROW. It rebuilds the whole row dict on - entry, so it is O(existing) per call — fine once, quadratic in a loop. The IG runner used to - call it per POST, which was survivable only because the cap was 5000; against `MAX_UT_IG_ROWS` - that same loop is hundreds of millions of dict copies and the automation simply never - finishes. Raising a cap and batching the writer are ONE change, not two. (W19-C.) - - ⚠ `capped` IS ITS OWN COUNT, deliberately not folded into `skipped`. They are different - facts: `skipped` means "this row had no key, so it could not be upserted" — a property of the - DATA, and usually benign. `capped` means "this table is full and the run is now losing rows" — - a property of the SYSTEM, and never benign. One number for both meant a table hitting its - ceiling was indistinguishable from a page with a few blank cells, which is how a time series - stops silently. The runners turn any `capped` into a `partial` run that NAMES the table. - """ - cap = MAX_UT_ROWS if cap is None else int(cap) - rows = {str(k): dict(v or {}) for k, v in (existing or {}).items()} - counts = {"inserted": 0, "updated": 0, "unchanged": 0, - "skipped": 0, "duplicates": 0, "orphans": 0, "capped": 0} - by_key, dupe_ids = {}, set() - for rid, row in rows.items(): - kv = str(row.get(key_field, "") or "").strip() - if not kv: - continue - if kv in by_key: - dupe_ids.add(rid) # a pre-existing duplicate: first id wins, second left - continue - by_key[kv] = rid - next_id = max((int(r) for r in rows if str(r).isdigit()), default=0) + 1 - seen_keys, incoming_dupes = set(), 0 - for src in incoming or []: - kv = str((src or {}).get(key_field, "") or "").strip() - if not kv: - counts["skipped"] += 1 # no key -> cannot be upserted; never guessed - continue - if kv in seen_keys: - incoming_dupes += 1 # the SOURCE listed it twice; first wins - continue - seen_keys.add(kv) - rid = by_key.get(kv) - if rid is None: - if len(rows) >= cap: - counts["capped"] += 1 # LOUD: the runner turns this into a partial run - continue - rid = str(next_id) - next_id += 1 - rows[rid] = dict(src) - by_key[kv] = rid - counts["inserted"] += 1 - continue - before = rows[rid] - # ⛔ D-116's PRECEDENCE RULE, applied per ROW because provenance is a property of the row. - # `held` counts the cells a lower-provenance source was refused, so the run can SAY it - # rather than quietly doing the right thing — a protection nobody is told about is - # indistinguishable from a source that happened to agree. - keep = set(protect(before, src) or ()) if protect else set() - use = {k: v for k, v in src.items() if k not in keep} if keep else src - if keep: - counts["held"] = counts.get("held", 0) + sum( - 1 for k in keep if k in src and str(before.get(k, "")) != str(src.get(k))) - changed = {k: v for k, v in use.items() if str(before.get(k, "")) != str(v)} - if changed: - before.update(use) - counts["updated"] += 1 - else: - counts["unchanged"] += 1 - counts["duplicates"] = incoming_dupes + len(dupe_ids) - counts["orphans"] = sum( - 1 for kv, rid in by_key.items() if kv not in seen_keys and rid not in dupe_ids) - return rows, counts - - -def dedupe_canonical_rows(existing, key_field, newest_by=""): - """Collapse duplicate logical rows while preserving the lowest stable row id. - - Canonical entity tables use this before every upsert. Snapshot tables deliberately do not: - repeated shortcodes there are new timestamped observations, not duplicate posts. Values are - taken newest-first and then filled from older rows, so a sparse fresh projection does not - erase a field an earlier row knew. - """ - rows = {str(k): dict(v or {}) for k, v in (existing or {}).items()} - groups = {} - for rid, row in rows.items(): - identity = str(row.get(key_field) or "").strip() - if identity: - groups.setdefault(identity, []).append((rid, row)) - removed = 0 - for members in groups.values(): - if len(members) < 2: - continue - keep = min((rid for rid, _row in members), key=lambda r: (not r.isdigit(), int(r) if r.isdigit() else r)) - ordered = sorted(members, key=lambda item: str(item[1].get(newest_by) or ""), reverse=True) \ - if newest_by else members - merged = {} - for _rid, row in ordered: - for key, value in row.items(): - if key not in merged or str(merged.get(key) or "").strip() == "": - merged[key] = value - rows[keep] = merged - for rid, _row in members: - if rid != keep: - rows.pop(rid, None) - removed += 1 - return rows, removed - - -# --------------------------------------------------------------------------------------------- -# USER TABLES — read/write through the RUNTIME (never core.user_tables' module-global key) -# --------------------------------------------------------------------------------------------- - -def _ut_slug(label): - s = re.sub(r"[^a-z0-9]+", "_", str(label or "").strip().lower()).strip("_") - return (s or "table")[:40] - - -def ut_all(rt): - try: - return dict(rt.get(UT_STORE_KEY) or {}) - except Exception: - return {} - - -def disable_for_table(rt, table_key, note="target database deleted"): - """Wave 21 (item 6a, C3): a deleted table's automations are DISABLED loudly, never deleted. - - The definition survives with `schedule.enabled = False` + a `statusNote`, so the rail still - shows what existed and why it stopped — silently deleting a user's automation because its - target died would read as data loss. Returns the ids it touched.""" - key = str(table_key or "") - touched = [] - - def _up(cur): - for aid, d in (cur or {}).items(): - if isinstance(d, dict) and (d.get("config") or {}).get("targetTable") == key: - sch = d.get("schedule") - if not isinstance(sch, dict): - sch = d["schedule"] = {} - sch["enabled"] = False - d["statusNote"] = note - touched.append(str(aid)) - return cur - - if key: - _store_update(rt, _up, flush="sync") - return touched - - -def ut_get(rt, key): - return ut_all(rt).get(str(key)) - - -def retire_automation_stage_fields(rt, tables=None): - """Delete obsolete Board-only fields and their hidden cells, never user fields. - - The authoritative selector is the engine's own ``automation.stageField`` / ``cyclesField`` - metadata — labels such as “Stage” are ordinary user vocabulary and are not touched. Old - generated timestamp/cycle cells are cleared too, including rows whose field definition was - removed by an interrupted earlier migration. Re-running this migration is a no-op. - - ⭐ WAVE 29 (W29-T01) — ``tables`` LETS A CALLER LEND ITS SNAPSHOT. The SCAN below is - O(all row-cells in the tenant) over a bucket whose documented ceiling is 35.8 MB / ~1.4 s to - deep-copy (see this module's header), and `GET /automations` was paying for THREE independent - copies of it per request. The scan is read-only, so borrowing the caller's copy is free; the - WRITE below still goes through `rt.update`, which re-reads under the store lock, so a lent - snapshot can never be the thing that gets written back. - """ - tables = ut_all(rt) if tables is None else tables - stage_keys = set() - for table in tables.values(): - if not isinstance(table, dict): - continue - for field in table.get("fields") or []: - auto = field.get("automation") if isinstance(field, dict) else None - if isinstance(auto, dict) and (auto.get("stageField") or auto.get("cyclesField")): - stage_keys.add(str(field.get("key") or "")) - for row in (table.get("rows") or {}).values(): - for key in (row or {}): - text = str(key) - if text.startswith("stage_auto_"): - stage_keys.add(text.removesuffix("_at").removesuffix("_cycles")) - stage_keys.discard("") - if not stage_keys: - return {"tables": 0, "fields": 0, "cells": 0} - - changed = {"tables": set(), "fields": 0, "cells": 0} - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - for table_key, table in cur.items(): - if not isinstance(table, dict): - continue - kept, removed = [], set() - for field in table.get("fields") or []: - auto = field.get("automation") if isinstance(field, dict) else None - key = str(field.get("key") or "") if isinstance(field, dict) else "" - if key in stage_keys and isinstance(auto, dict) and \ - (auto.get("stageField") or auto.get("cyclesField")): - removed.add(key) - changed["fields"] += 1 - continue - kept.append(field) - if removed: - table["fields"] = kept - changed["tables"].add(str(table_key)) - for row in (table.get("rows") or {}).values(): - if not isinstance(row, dict): - continue - for stage_key in stage_keys: - for key in (stage_key, stage_key + "_at", stage_key + "_cycles"): - if key in row: - row.pop(key, None) - changed["cells"] += 1 - changed["tables"].add(str(table_key)) - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") - return {"tables": len(changed["tables"]), "fields": changed["fields"], - "cells": changed["cells"]} - - -def bind_unbound_fields(rt): - """C8's migration (wave 22, stated choice): existing automation bags WITHOUT a `flowId` - are bound to the definition that already writes them — matched on the (targetTable, - fieldKey) pair the definition carries, which is the binding W21-C2 established. A bag no - definition references is DISABLED with the reason on it rather than deleted or guessed: - the column was already dead (nothing runs a column no flow names), and now it says so. - Returns `(bound, disabled)`; costs zero store commits when there is nothing to migrate.""" - by_binding = {} - for aid, d in all_definitions(rt).items(): - cfg = d.get("config") or {} - if cfg.get("fieldKey") and cfg.get("targetTable"): - by_binding[(str(cfg["targetTable"]), str(cfg["fieldKey"]))] = str(aid) - plan = {} - for tk, t in ut_all(rt).items(): - for f in (t.get("fields") or []): - a = f.get("automation") - # An already-DISABLED bag is a decision this migration made on a previous pass — - # re-planning it every call would turn the once-per-process sweep into a write - # per read. - if isinstance(a, dict) and not a.get("flowId") and not a.get("stageField") \ - and not a.get("disabled"): - plan[(tk, str(f.get("key")))] = by_binding.get((tk, str(f.get("key")))) - if not plan: - return 0, 0 - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - for (tk, fk), aid in plan.items(): - for f in ((cur.get(tk) or {}).get("fields") or []): - if f.get("key") == fk and isinstance(f.get("automation"), dict): - if aid: - f["automation"]["flowId"] = aid - else: - f["automation"]["disabled"] = True - f["automation"]["statusNote"] = ( - "not bound to any flow. Create an automation for this column " - "or delete it") - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") - bound = sum(1 for v in plan.values() if v) - return bound, len(plan) - bound - - -def ut_key_for(label, key=None): - """The table key a label resolves to. Factored out of `ut_ensure` so the DRY-RUN path (which - must not create anything) resolves the same key by construction rather than by copying the - derivation and drifting from it.""" - k = str(key or (UT_PREFIX + _ut_slug(label))) - return k if k.startswith(UT_PREFIX) else UT_PREFIX + k - - -#: Machine names, mirrored from `core.user_tables.MACHINE_OWNERS`. A LOCAL literal for the same -#: reason `UT_FIELD_TYPES` is one over there — this module must stay importable without dragging -#: `core` into the API's boot path — and the two are held in step by a gate check. -MACHINE_OWNERS = ("automation", "scheduler") - - -def ut_ensure(rt, label, fields, username="automation", key=None, flow_tag="", - record_mode="", lock_fields=False, tables=None): - """Create the table if it is missing; return its key. Idempotent — a re-run of an automation - that owns a table must not spawn `ut_x_2`, so the key is DERIVED from the label (or given) - and an existing table with that key is adopted, not duplicated. - - ⭐⭐ WAVE 31 · T30 — `tables` LETS A CALLER LEND THE SNAPSHOT IT IS ALREADY HOLDING, and it is - the SKIP TEST below that it pays for. `rt.get` deep-copies the whole tenant document on every - call (documented ceiling 35.8 MB / ~1.4 s), so a caller that ensures FOUR tables in one pass - paid four copies to answer four questions about one document. Owner item 7, verbatim: *"It still - takes forever to change the Config for automation as well. It just say Saving..."* — measured - **7,912 ms** for one save on `4258a93`. - - ⚠ THE LEND IS READ-ONLY AND SAFE BY INSPECTION, not by hope — the same argument - `retire_automation_stage_fields` (this module, same parameter name, same contract) already - makes: only the `have` lookup below reads it, and the WRITE still goes through `rt.update`, - whose `_up` re-reads the live document under the store lock. **A lent snapshot can therefore - never be the thing that gets written back**, and the worst a stale one can do is spend a commit - that would have been skipped — never write a wrong value. Callers that ensure two tables - under the SAME key in one pass pass `tables=None` for the second (see `_spawn_presets`). - - Fields are MERGED, never replaced: a user who added a column to an automation's table keeps - it, and a new source column joins on the next run. - - `flow_tag` (wave 22, C7/item 5): every field THIS call adds is stamped - `automation: {flowId: }` — the pre-set columns an IG automation spawns carry their - provenance. Fields already on the table keep whatever tag they have (first flow wins; - shared tables like ut_ig_snapshots are fed by many flows and the tag is provenance, not - ownership). - - ⛔ AND IT STAMPS A HUMAN OWNER, WHICH IT DID NOT (wave 20, item 3). `createdBy` was whoever - or WHATEVER ran the automation, so the same table belonged to a person if its first run was - manual and to `"scheduler"` if the schedule got there first — and `user_tables.may_open` - admits only the creator or an admin, so **whether you could open your own Instagram - snapshots depended on a race you never saw**. The owner is now the automation's creator, and - a table already stamped with a machine name is ADOPTED the next time a run knows a human - one. Adoption is a repair, not a widening: the automation's creator is the person who asked - for the table in the first place. - """ - key = ut_key_for(label, key) - # ⭐ WAVE 26 — THE MIGRATION RIDES THE WRITE PATH, and that placement is the point. - # - # `ut_ensure` MERGES fields by key and never re-types an existing column, so on its own it - # would leave every table already in production on the old `text` schema forever while new - # ones got the honest types — the split schema R3's note warned about, arriving through the - # very function the note was written on. Migrating here means a table is brought forward - # immediately BEFORE anything appends to it, so no caller has to remember anything and no - # tenant is left behind by a script nobody ran. - # ⚠ Cheap by construction: `migrate_ig_tables` returns without a write when the table is - # already current, which after the first run is every call. - if key and {f.get("key") for f in (fields or [])} & set(PRESET_PROFILE_KEYS): - try: - migrate_ig_tables(rt, log=lambda *_a: None, only=key) - except Exception: # noqa: BLE001 - # A migration that cannot run must not stop the automation from writing its rows. - # The old schema still reads; a refused write loses the pull we just paid for. - pass - wanted = [] - for raw_field in (fields or []): - field = dict(raw_field) - if flow_tag or lock_fields: - automation = dict(field.get("automation") or {}) - if flow_tag: - automation.setdefault("flowId", str(flow_tag)) - if lock_fields: - automation["preset"] = True - field["automation"] = automation - wanted.append(field) - created = _iso() - human = username if username and username not in MACHINE_OWNERS else "" - - # ⚠ SKIP THE WRITE WHEN NOTHING WOULD CHANGE. Without this, every re-run spends a store - # commit re-writing an identical definition — against a 20 s flush floor and a 256/hr repo - # budget, an idempotent helper that always writes is the same defect as a per-row insert. - have = ut_get(rt, key) if tables is None else tables.get(str(key)) - have_fields = {str(f.get("key") or ""): f for f in ((have or {}).get("fields") or [])} - missing = [f for f in wanted if f.get("key") not in have_fields] - missing_locks = [f for f in wanted - if lock_fields and f.get("key") in have_fields - and (not isinstance(have_fields[f.get("key")].get("automation"), dict) - or have_fields[f.get("key")]["automation"].get("preset") is not True)] - if (have is not None and not missing and not missing_locks - and not (record_mode and have.get("recordMode") != record_mode) - and not (human and (have.get("createdBy") or "") in MACHINE_OWNERS)): - return key - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - t = cur.get(key) - if t is None: - if len(cur) >= MAX_UT_TABLES: - return cur - cur[key] = {"key": key, "label": str(label)[:60], "source": "Automation", - "createdBy": username, "created": created, - "fields": wanted, "rows": {}} - if record_mode: - cur[key]["recordMode"] = record_mode - return cur - have = {f.get("key") for f in (t.get("fields") or [])} - for f in wanted: - if f.get("key") not in have: - t.setdefault("fields", []).append(f) - have.add(f.get("key")) - elif lock_fields: - stored = next((g for g in (t.get("fields") or []) - if g.get("key") == f.get("key")), None) - if stored is not None: - automation = dict(stored.get("automation") or {}) - if flow_tag: - automation.setdefault("flowId", str(flow_tag)) - automation["preset"] = True - stored["automation"] = automation - # ADOPTION: a machine name is not an owner. It never overwrites a human one. - if human and (t.get("createdBy") or "") in MACHINE_OWNERS: - t["createdBy"] = human - # An automation's table SAYS an automation owns it — the nav badge reads from this. - t["source"] = t.get("source") or "Automation" - if record_mode: - t["recordMode"] = record_mode - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") - return key - - -def ut_write_rows(rt, key, rows): - """ONE store update for the WHOLE row set — the flush-ceiling rule (see the module header).""" - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - t = cur.get(key) - if t is not None: - t["rows"] = rows - return cur - rt.update(UT_STORE_KEY, _up, flush="sync") - - -def ut_set_cell(rt, key, row_id, field_key, value, extra_rows=None): - """Write one automation-owned cell (+ optional whole extra tables) in ONE update.""" - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - t = cur.get(key) - if t is not None: - t.setdefault("rows", {}).setdefault(str(row_id), {})[field_key] = value - for k, rws in (extra_rows or {}).items(): - tt = cur.get(k) - if tt is not None: - tt["rows"] = rws - return cur - rt.update(UT_STORE_KEY, _up, flush="sync") - - -IG_FIELD_DESCRIPTIONS = { - "alt_text": "Accessibility text attached to the Instagram post.", - # ⚠ WIDENED 2026-08-10, because the column gained a second writer and the old sentence would - # have made it lie. It was written for the anonymous rung, whose numbers are genuinely ROUNDED - # ("10.4K followers"). A DISCOVERY observation is exact-looking and STALE instead — read off - # the vendor's pre-collected corpus at a collection time we are not told. Both mean the same - # thing to a reader ("do not treat this as an exact count taken at Pulled at"), and a - # description that named only the first would have quietly excluded the second. - "approx": "Checked when the counts are rounded, or were read from a pre-collected corpus " - "rather than measured at the time shown.", - "avg_comments_12": "Average comments across the 12 most recent captured posts.", - "avg_engagement": "Average engagement rate reported for the profile.", - "avg_likes_12": "Average likes across the 12 most recent captured posts.", - "avg_plays_12": "Average plays across the 12 most recent captured posts.", - "avg_views_12": "Average views across the 12 most recent captured posts.", - "views": "View count for the post, as Instagram displays it.", - "video_duration": "Length of the video in seconds.", - "comments_disabled": "Checked when the creator has turned comments off for this post.", - "plays": "Times the video started playing, including replays.", - "bio": "Biography shown on the Instagram profile.", - "bio_hashtags": "Hashtags listed in the profile biography.", - "business_category": "Instagram's business category for the account.", - "caption": "Caption published with the Instagram post.", - "category": "Instagram's category for the profile.", - "comment_key": "Unique ID for the captured comment record.", - "commented_at": "Date the comment was posted.", - "comments": "Comment count reported for the post.", - "comments_captured": "Number of distinct comment records linked to this profile.", - "comments_link": "Comment records linked to this record.", - "country_code": "Country code reported for the profile.", - "created_by": "User whose automation first added this lead.", - "enriched_at": "Date the profile was last enriched.", - # ⭐ ITEM 16. The description says GUESS out loud, because the column is one and the number - # beside it is the only thing that says how much of one. - "location_guess": "Most common city tagged across this profile's captured posts - a guess, " - "not a stated location.", - "location_confidence": "Share of this profile's geotagged posts that agree on the guessed " - "city.", - "external_url": "Website linked from the Instagram biography.", - "external_url_title": "Title Instagram shows for the biography link.", - "fbid": "Facebook ID associated with the Instagram account.", - "first_found": "Date an automation first found this profile.", - "followers": "Latest reported follower count.", - "following": "Latest reported number of accounts followed.", - "found_count": "Number of times discovery automations found this profile.", - "full_name": "Display name shown on the Instagram profile.", - "handle": "Instagram username without the @ symbol.", - "has_channel": "Checked when the profile has an Instagram channel.", - "hashtags": "Hashtags extracted from the post caption.", - "highlights_count": "Total story highlight collections reported for the profile.", - "ig_id": "Instagram's internal ID for the account.", - "influencer_key": "Normalized handle linking this row to its profile.", - "is_business": "Checked when Instagram marks the account as a business.", - "is_joined_recently": "Checked when Instagram marks the account as recently joined.", - "is_private": "Checked when the Instagram profile is private.", - "is_professional": "Checked when Instagram marks the account as professional.", - "last_found": "Date an automation most recently found this profile.", - "likes": "Like count reported for this record.", - "measured_at": "Date engagement metrics on this post were last read.", - "measurements_captured": "Number of measurement rows stored for this post.", - "paid_partnership": "Checked when Instagram marks the post as a paid partnership.", - "partner": "Brand named in the post's paid partnership metadata.", - "partner_id": "Partner ID reported for the Instagram profile.", - "platform": "Social network for this profile.", - "plays": "Video plays, including repeat plays, reported for the post.", - "post_link": "Post record linked to this measurement or comment.", - "post_measurements_captured": "Number of distinct post measurements linked to this profile.", - "post_snapshot_key": "Unique ID for this post measurement.", - "post_snapshots_link": "Engagement measurement rows linked to this record.", - "posted_at": "Date the Instagram post was published.", - "posts_captured": "Number of distinct post records captured for this profile.", - "posts_count": "Total posts reported for the Instagram profile.", - "posts_link": "Post records linked to this profile.", - "profile_name": "Profile name returned by the data provider.", - "profile_reads": "Number of profile snapshots stored for this profile.", - "profile_snapshots_link": "Profile snapshot rows linked to this profile.", - "profile_url": "Direct URL to the Instagram profile.", - "pronouns": "Pronouns shown on the Instagram profile.", - "pulled_at": "Date this snapshot was collected.", - "related_accounts": "Accounts Instagram suggests alongside this profile.", - "replies": "Reply count reported for the comment.", - "shortcode": "Instagram shortcode that uniquely identifies the post.", - "source_payload": "Full source response for this record.", - "snapshot_key": "Unique ID for this profile snapshot.", - "source": "Method or provider used to read this profile.", - "tagged_location": "Location tagged on the post; it is not the creator's residence.", - "text": "What the comment says. The commenter's name is not stored as a column.", - "type": "Instagram media type: image, video, or carousel.", - "url": "Direct URL to the Instagram post.", - "verified": "Checked when Instagram marks the profile as verified.", -} - - -#: ⭐ WAVE 29 (R2) — TIKTOK'S OWN DESCRIPTIONS, and the reason this exists rather than reusing the -#: map above is one line of `field_def`: it defaults `description` to -#: `IG_FIELD_DESCRIPTIONS.get(key)`. Nine `ut_tt_*` columns share a KEY with an Instagram column -#: (`shortcode`, `type`, `url`, `verified`, `caption`, `likes`, `comments`, `replies`, -#: `tagged_location`), so a TikTok video's Type column would have shipped explaining "Instagram -#: media type" — a wrong sentence in the header tooltip of a column nobody would think to check. -#: ⚠ A KEY ABSENT HERE GETS NO DESCRIPTION AT ALL, deliberately: silence is honest, and inheriting -#: the Instagram sentence is the failure this map exists to prevent. -TT_FIELD_DESCRIPTIONS = { - "platform": "Network this handle is on.", - "handle": "TikTok @name; the unique account handle.", - "full_name": "Display name shown on the TikTok profile.", - "tt_id": "TikTok's own numeric id for the account.", - "profile_url": "Direct URL to the TikTok profile.", - "bio": "Profile biography text.", - "external_url": "Link in the TikTok bio.", - "verified": "Checked when TikTok marks the account as verified.", - "is_private": "Checked when the account is private.", - "followers": "Follower count at the time of the pull.", - "following": "Accounts this profile follows.", - "posts_count": "Videos published by this account.", - "likes_received": "Total likes this account's videos have received.", - "avg_engagement": "Average engagement rate, stored as a percentage.", - "comment_engagement": "Comment engagement rate, stored as a percentage.", - "like_engagement": "Like engagement rate, stored as a percentage.", - "is_business": "Approximate: set when TikTok flags the account as a commerce user.", - "country_code": "Two-letter country code reported for the account.", - "predicted_lang": "Language TikTok predicts for this account.", - "account_created_at": "When the TikTok account itself was created.", - "region": "Region reported for the account.", - "shortcode": "TikTok's numeric id for the video; unique per post.", - "type": "TikTok post type: video or image.", - "url": "Direct URL to the TikTok post.", - "caption": "Post description text.", - "posted_at": "When the creator published the post.", - "likes": "Likes reported for the post.", - "comments": "Comment count reported for the post.", - "views": "Play count TikTok reports for the video.", - "shares": "Times the post was shared.", - "saves": "Times the post was saved to a collection.", - "video_duration": "Video length in seconds.", - "hashtags": "Hashtags used in the post.", - "tagged_location": "Commerce location reported for the post; it is not the creator's home.", - "influencer_key": "Handle of the account this record belongs to.", - "comment_key": "Unique ID for this comment.", - "commented_at": "When the comment was posted.", - "text": "What the comment says. The commenter's name is not stored as a column.", - "replies": "Reply count reported for the comment.", - "snapshot_key": "Unique ID for this profile snapshot.", - "post_snapshot_key": "Unique ID for this post measurement.", - "pulled_at": "When this measurement was read.", - "enriched_at": "When this row was last enriched.", - "source": "Method or provider used to read this record.", - "source_payload": "Full source response for this record.", -} - - -def field_def(text_key, label, ftype="text", **extra): - """One machine-spawned column definition. - - ⭐ 2026-08-07 — `**extra` carries the per-field DECLARATIONS the preset set needs (`pinned`, - `profile`), and it stays ONE constructor rather than growing a second for "special" fields. - ⛔ Every key passed here must survive `core.user_tables._clean_field` UNCHANGED — `verify_api`'s - W25-1 section pins the drift at ZERO, so a key the validator drops or rewrites turns the whole - preset list red rather than failing quietly ([[default-must-pass-its-own-guard]]). - - ⭐ WAVE 25 — `editRole` IS DECLARED HERE, and adding it closes a bypass rather than adding a - feature. `ut_ensure` writes these dicts STRAIGHT into the `user_tables` bucket, so - `core.user_tables._clean_field` — the single validator every field created through the ordinary - door passes — has never judged a single field this engine spawned. The difference was exactly - one key: `_clean_field` emits `editRole: 'admins'` and this did not, so `_clean_field(f) != f` - for every automation column in the product. - ⚠ NOTHING CHANGES BEHAVIOURALLY — `may_edit_field` asks `editRole == 'everyone'`, and an - absent key was already not that, so the bypass was fail-closed and therefore silent. It is - stated now, and `verify_automation` asserts the equality field-by-field, so the next key - `_clean_field` grows cannot go unnoticed here ([[default-must-pass-its-own-guard]]). - """ - description = " ".join(str(extra.pop( - "description", IG_FIELD_DESCRIPTIONS.get(text_key, "")) or "").split()) - out = {"key": text_key, "label": label, "type": ftype, "source": "overlay", - "editRole": "admins"} - if description: - out["description"] = description - out.update(extra) - return out - - -def tt_field_def(text_key, label, ftype="text", **extra): - """One `ut_tt_*` column. `field_def` with TikTok's description map bound (wave 29, R2). - - ⛔ `description` IS ALWAYS SUPPLIED, even when blank, so `field_def`'s Instagram default can - never be reached from here. An explicit empty string makes `field_def` omit the key, which is - the same shape an undescribed column already has — the point is that the sentence a TikTok - column carries is one somebody wrote about TikTok, or none at all. - """ - extra.setdefault("description", TT_FIELD_DESCRIPTIONS.get(text_key, "")) - return field_def(text_key, label, ftype, **extra) - - -# --------------------------------------------------------------------------------------------- -# DEFINITIONS — validation and the bucket's read/write half -# --------------------------------------------------------------------------------------------- - -def _s(v, n=200): - return str(v if v is not None else "")[:n] - - -#: ⭐ WAVE 26 · C5 / R2 — HOW MANY POSTS ONE PULL MAY KEEP, and the ceiling is the VENDOR'S. -#: -#: ⛔ MEASURED 2026-08-05 and recorded at `_bd_posts_count`: **a Profiles row carries the TOP 12 -#: posts — a cap, not a count.** Asking for more does not fetch more; it just makes the config -#: disagree with what the run can possibly do. -#: ⚠ THIS REPLACES A SILENT `max(1, min(n, 200))` AT TWO CALL SITES, and the clamp was the defect -#: rather than the number: a person typing 50 got a stored 50, a UI that read back 50, and twelve -#: posts — with nothing anywhere saying why. A control that accepts a value it cannot honour is -#: worse than one that refuses it, because the refusal is the only place the ceiling can be -#: taught. So this REFUSES, and the sentence names the cap and who set it. -#: -#: ⭐⭐ 2026-08-09 — RAISED TO 30 (owner: *"move the max limit to 30 posts per profile"*), and the -#: paragraph above needed correcting to do it honestly: **12 was OUR ROUTE'S cap, not the -#: vendor's.** Re-measured the same day — the PROFILE record still returns 12 however many you ask -#: for (asked 40, got 12), but the documented discover-by-url route answered `num_of_posts: 30` -#: with exactly 30 rows in 90 s (23 Reels + 7 Carousels, @theresalearns). The ceiling was a -#: property of the call we happened to make. -#: ⛔ SO THE NUMBER MOVED AND THE CAPTURE MUST FOLLOW. Until `bd_profile_posts` is wired ahead of -#: the views top-up, a `maxPosts` above 12 is honoured by the CONFIG and bounded by the PROFILE -#: read at run time — the exact accept-what-you-cannot-honour shape this constant exists to -#: prevent, now surviving in one place instead of two. It is recorded rather than hidden, and it -#: is why `clean_post_groups` bounds a group by `maxPosts` rather than by 12. -MAX_POSTS_PER_PULL = 30 -DEFAULT_POSTS_PER_PULL = 10 - -#: ⭐⭐ THE WINDOW THE `avg_*_12` PRESET ROLLUPS AVERAGE OVER — ITS OWN CONSTANT, deliberately -#: NOT `MAX_POSTS_PER_PULL`. -#: -#: ⛔ THEY WERE THE SAME NUMBER AND THAT WAS A LATENT BUG, caught by the gate the moment the cap -#: moved: four columns are NAMED `avg_views_12` and LABELLED "Avg views · last 12 posts", so -#: raising the capture cap to 30 silently made every one of them average thirty posts under a -#: label that says twelve. Nobody would have looked at those columns again to check. -#: ⚠ Raising the CAPTURE ceiling and redefining an EXISTING named measure are two different -#: decisions, and only the first one was made. To widen the average, change this constant AND the -#: four keys and labels together — a column whose meaning changes underneath its own name is -#: worse than one that is merely narrow. -AVG_WINDOW_POSTS = 12 - - -#: The post kinds a group filter may name — exactly what `_bd_type` maps a vendor row onto, so a -#: filter cannot ask for a category that can never match a stored row. -POST_TYPES = ("video", "image", "carousel") -#: ⭐ W29-T09 — the words a PERSON reads for those three keys, server-owned for the same reason -#: `KIND_LABELS` and `TRIGGER_LABELS` are. The owner asked for *"the last 12 reels"*, and `video` -#: is the stored key: a client that translated it locally would be a second copy of this -#: vocabulary, free to drift the day a fourth kind appears or a name changes. -#: ⚠ "Reels & videos", not "Reels": `_bd_type` maps every non-image, non-carousel post here, so a -#: label naming only reels would over-promise on a plain video post. -POST_TYPE_LABELS = {"video": "Reels & videos", "image": "Photos", "carousel": "Carousels"} - - -def clean_post_groups(raw, max_posts): - """`(groups, error)` for `config.postGroups` — "last N reels, last M images". - - ⭐ 2026-08-09 (owner: *"not just last 12 but by group also"*). Shape: - `[{"type": "video", "limit": 12}, {"type": "image", "limit": 6}]`. - - ⛔ ABSENT IS OFF, and off must stay the default forever: an enrich action stored before today - carries no such key, and inventing a filter for it would silently start DROPPING posts those - automations have always captured. Empty list and missing are the same answer. - - ⚠ A MALFORMED GROUP IS REFUSED, not dropped — the opposite of `tier`/`noFallback`, and the - difference is legitimate. Those are RETIRED keys that live in stored data, so refusing them - would 400 old automations forever (D-65). This key is NEW: nothing stored can carry a broken - one, so the only way to see one is a person typing it, and a filter that silently ignores the - type you asked for is how you end up paying for reels and storing carousels. - """ - if raw in (None, "", [], {}): - return [], None - if not isinstance(raw, list): - return None, "the post groups have to be a list of {type, limit} entries" - out, seen = [], set() - for item in raw: - if not isinstance(item, dict): - return None, "each post group has to be a {type, limit} entry" - t = str(item.get("type") or "").strip().lower() - if t not in POST_TYPES: - return None, (f"'{t or 'blank'}' is not a post type. Use one of " - f"{', '.join(POST_TYPES)}") - if t in seen: - return None, f"the post groups name '{t}' twice. One limit per type" - seen.add(t) - try: - lim = int(item.get("limit")) - except (TypeError, ValueError): - return None, f"how many {t} posts to keep has to be a whole number" - if lim < 1: - return None, f"a {t} group has to keep at least one post" - # ⛔ BOUNDED BY WHAT THE RUN ACTUALLY BUYS. A group asking for 30 when the pull captures - # 12 is not an error a person can act on — it is a promise the run cannot keep — so it is - # refused HERE, where the number they typed is still on the screen in front of them. - if lim > max_posts: - return None, (f"this enrichment captures {max_posts} posts per profile, so a {t} " - f"group cannot keep {lim}. Raise the post count first") - out.append({"type": t, "limit": lim}) - return out, None - - -def clean_max_posts(raw, default=DEFAULT_POSTS_PER_PULL, submitted=True): - """`(maxPosts, error)` — refuses out-of-range rather than clamping into range. - - ⛔ `submitted=False` CLAMPS INSTEAD OF REFUSING, and that asymmetry is the whole reason this - takes a flag. **Every automation stored before wave 26 carries `maxPosts: 24`** — the old - clamp's default, which this cap now forbids. If an inherited value were refused the same way - a typed one is, every one of those automations would 400 on its next Save **forever**, for a - number nobody on that screen chose or can see. That is D-65's lesson exactly: a validator - that refuses stored data does not protect the user from it, it locks them out of their own - automation. - So: a value the panel SENT is the user asking for it, and gets the sentence. A value merely - INHERITED gets silently brought inside the cap it was already effectively subject to at the - vendor — the run was returning 12 either way ([[default-must-pass-its-own-guard]]). - """ - if raw is None or (isinstance(raw, str) and not raw.strip()): - return default, None - try: - n = int(raw) - except (TypeError, ValueError): - if not submitted: - return default, None - return None, "the post limit must be a whole number" - if not submitted: - return max(1, min(n, MAX_POSTS_PER_PULL)), None - if n < 1: - return None, "keep at least one post per profile, or turn the post capture off" - if n > MAX_POSTS_PER_PULL: - return None, (f"a profile pull returns at most {MAX_POSTS_PER_PULL} posts. That is the " - f"vendor's cap, not ours, so {n} would store the same " - f"{MAX_POSTS_PER_PULL} and read as more") - return n, None - - -def clean_config(kind, raw, previous=None): - """Validate a kind's config. Returns `(config, error)`; error is a user-facing sentence. - - ⚠ THE NODE-SWITCH FLAGS FALL BACK TO `previous` WHEN THE KEY IS ABSENT, and that is a rail - rather than a nicety. `patch` replaces the whole config, and the canvas's config panels do not - edit `postMetrics` / `commentMetrics` / `dryRun` — those are node SWITCHES. So a plain "Save" - from a panel that never knew about them would silently turn off the dry run, or turn ON a - per-post purchase: a save that quietly changes what the automation costs. Absent ⇒ keep; - present ⇒ take it, including `false`. - """ - raw = raw if isinstance(raw, dict) else {} - prev = previous if isinstance(previous, dict) else {} - - def flag(name): - return bool(raw[name]) if name in raw else bool(prev.get(name)) - if kind == "plain": - # A plain automation has no machine step, so it only needs the database its records walk. - # - # ⚠ THE TARGET IS OPTIONAL ON PURPOSE. An automation triggered by `record_updated` on - # `ut_foo` already knows its table from the trigger — `_flow_table` resolves target-else- - # trigger — so requiring a second copy of that fact here would be a mandatory field with - # exactly one legal answer, and a Save that refuses until you retype what you just picked. - # There is no `dryRun`: dry-run means "run the machine step but do not write", and there - # is no machine step to run. - target = _s(raw.get("targetTable"), 60).strip() - if target and not target.startswith(UT_PREFIX): - target = UT_PREFIX + target - return {"targetTable": target, - "targetLabel": _s(raw.get("targetLabel"), 60).strip()}, None - if kind == "scrape_db": - url = _s(raw.get("url"), 2000).strip() - if not url: - return None, "a source URL is required" - try: - guard(url) - except Refused as e: - return None, str(e) - extract = raw.get("extract") if raw.get("extract") in EXTRACTS else "table" - fmap = {} - for k, v in list((raw.get("fieldMap") or {}).items())[:60]: - tk = re.sub(r"[^a-z0-9_]+", "_", _s(v, 60).strip().lower()).strip("_") - if tk: - fmap[_s(k, 120)] = tk[:60] - if not fmap: - return None, "map at least one source column to a field" - key_field = _s(raw.get("keyField"), 60).strip() - if key_field not in fmap.values(): - return None, "the key field must be one of the mapped fields" - target = _s(raw.get("targetTable"), 60).strip() - if target and not target.startswith(UT_PREFIX): - target = UT_PREFIX + target - return {"url": url, "extract": extract, - "tableIndex": max(0, min(int(raw.get("tableIndex") or 0), 50)), - "fieldMap": fmap, "keyField": key_field, - "targetTable": target, - "dryRun": flag("dryRun"), - "targetLabel": _s(raw.get("targetLabel"), 60).strip() or "Scraped table"}, None - if kind == "field_instagram": - table = _s(raw.get("targetTable"), 60).strip() - if not table.startswith(UT_PREFIX): - return None, "an Instagram automation runs against a blank database (ut_*)" - fkey = _s(raw.get("fieldKey"), 80).strip() - if not fkey: - return None, "pick the automation column this run writes into" - # ⛔⛔ `tier` AND `noFallback` ARE ACCEPTED AND IGNORED (wave 28 / R5, contract C2). - # They are read from nothing and written to nothing: a stored definition carrying either - # still SAVES — it simply loses them on the next write — and neither is ever a reason to - # refuse. That asymmetry is D-65's law and it is not squeamishness: refusing an unknown - # key would 400 every automation a tenant stored before this wave, forever, on a screen - # that gives them no way to remove it. Dropping a retired key is a migration; refusing it - # is an outage. - # ⚠ `clean_tier`/`TIERS`/`bd_ready` KEEP THEIR NAMES. They are VENDOR vocabulary - # (`brightdata` is a provider, and `verify_automation` fences the name), not the retired - # USER concept — renaming them would be a second, unrelated change wearing this one's - # justification. - # C5: absent ⇒ keep whatever is stored, like every other switch in this branch — a panel - # that does not edit the post count must not reset it to the default on Save. And an - # INHERITED value is clamped rather than refused; see `clean_max_posts`. - sent = "maxPosts" in raw - max_posts, perr = clean_max_posts( - raw.get("maxPosts") if sent else prev.get("maxPosts"), submitted=sent) - if perr: - return None, perr - return {"targetTable": table, "fieldKey": fkey, - "urlField": _s(raw.get("urlField"), 80).strip(), - # ⛔ THE ONE FLAG THAT MULTIPLIES THE BILL BY THE POST COUNT. Off unless asked - # for: the Profiles row carries post IDENTITY for free but NO engagement - # (measured 2026-08-05), so likes/comments cost one extra vendor record PER POST. - "postMetrics": flag("postMetrics"), - # The full Comments dataset can bill many rows per post. It is an explicit, - # independent opt-in and never follows the per-post switch automatically. - "commentMetrics": flag("commentMetrics"), - "dryRun": flag("dryRun"), - "maxPosts": max_posts}, None - # ⭐ WAVE 29 (D-9 / R1) — BOTH discovery kinds share this branch, because they ask the vendor - # the same question of two different corpora: a record ceiling, a predicate list, a join word - # and where to write. ⛔ THE ONLY DIFFERENCE IS THE DEFAULT TABLE, and it is resolved from the - # kind rather than hard-coded — a TikTok search falling back to `ut_ig_profile` would write - # TikTok rows into the Instagram family, which is precisely what R2's parallel family exists - # to prevent, and it would do it silently. - if kind in DISCOVERY_KINDS: - # ⭐ WAVE 30 · T05 — the inline tuple became the named one, and the two defaults below now - # come from `discovery_facts` rather than being spelled out here. They were correct; they - # were also the FIFTH copy of "which table does this kind write to", and the other four - # were the ones wave 29 forgot to update. - _, _disc_table, _disc_label, _ = discovery_facts(kind) - limit = int(raw.get("recordsLimit") or 0) - if limit < 1: - return None, ("say how many profiles to fetch. An UNBOUNDED discovery query is the " - "one shape the vendor refuses outright (NOT_ENOUGH_FUNDS)") - if limit > BD_MAX_RECORDS: - return None, f"a single discovery run may ask for at most {BD_MAX_RECORDS} profiles" - # ⚠ THE JOIN IS RESOLVED BEFORE THE PREDICATES ARE JUDGED, because it is part of what - # makes them broad or narrow (see `narrowing_refusal`). Validating them first and reading - # the operator afterwards is how the OR hole survived: the guard was handed the branches - # and never told they were a union. - join = "or" if str(raw.get("operator") or "").lower() == "or" else "and" - # ⭐ WAVE 32 · T46 (D-167) — AND THE KIND GOES WITH THEM. `clean_config` already knows which - # corpus this automation searches; passing it is what stops a `discover_tiktok` being built - # on the 16 Instagram fields TikTok's dataset does not carry — a search that does not error, - # returns nothing, and reads as "no such creators exist" after the money is spent. - preds, perr = clean_predicates(raw.get("predicates"), join, kind) - if perr: - return None, perr - target = _s(raw.get("targetTable"), 60).strip() or _disc_table - if not target.startswith(UT_PREFIX): - target = UT_PREFIX + target - seed, serr = clean_seed(raw.get("seed") if "seed" in raw else prev.get("seed")) - if serr: - return None, serr - return {"recordsLimit": limit, "predicates": preds, - "operator": join, - "targetTable": target, - "targetLabel": _s(raw.get("targetLabel"), 60).strip() or _disc_label, - # C6/R5: WHERE the conditions above came from, when they were derived. Stored so - # the surface can say "these were filled in from the 'Florists' view, over 12 - # records" instead of presenting them as if somebody typed them. - "seed": seed, - "dryRun": flag("dryRun")}, None - return None, f"unknown automation kind {kind!r}" - - -def clean_seed(raw): - """C6: validate `config.seed`. Returns `({}, None)` when there is none — a discovery filter - somebody typed by hand has no seed, and that is the ordinary case. - - ⚠ `derived` AND `basis` ARE STORED AS PROVENANCE, NOT AS A SECOND FILTER. R5 is explicit that - the derived conditions are "written into `config.predicates` as ordinary conditions — it is a - filling-in, not a parallel filter", so the RUN never reads this bag: it reads `predicates`, - like every other search. Keeping it means the surface can say where those rows came from, and - a user editing them freely is exactly what is supposed to happen. - """ - if raw in (None, "", {}): - return {}, None - if not isinstance(raw, dict): - return None, "the seed must be an object" - source = _s(raw.get("source"), 12).strip().lower() - if source not in SEED_SOURCES: - return None, f"{source or 'that seed'!r} is not one of: " + ", ".join(SEED_SOURCES) - table = _s(raw.get("table"), 60).strip() - if table and not table.startswith(UT_PREFIX): - return None, f"a seed reads a blank database (ut_*). {table!r} is not one" - basis = raw.get("basis") if isinstance(raw.get("basis"), dict) else {} - derived, _err = clean_predicates(raw.get("derived") or [], "and") - return {"source": source, "table": table, "id": _s(raw.get("id"), 80).strip(), - # ⛔ A MALFORMED `derived` IS DROPPED, NEVER A REFUSAL. This bag is a RECORD of what - # was suggested; the conditions that matter are already in `predicates` and were - # validated there. Refusing a Save because a stored provenance note aged badly would - # block the user from editing the very filter it describes. - "derived": derived or [], - "basis": {"rows": max(0, int(basis.get("rows") or 0)), - "fields": [f for f in (basis.get("fields") or []) - if isinstance(f, dict)][:SEED_MAX_PREDICATES], - "related": basis.get("related") - if isinstance(basis.get("related"), dict) else {}, - "note": _s(basis.get("note"), 200)}}, None - - -def clean_schedule(raw, previous=None): - """Validate `{cron, enabled}` and stamp `enabledAt` on the OFF→ON edge (the anchor `is_due` - measures from — without it, enabling a daily job fires it immediately).""" - raw = raw if isinstance(raw, dict) else {} - cron = _s(raw.get("cron"), 120).strip() or "0 6 * * *" - parse_cron(cron) # raises -> the route answers 400 - enabled = bool(raw.get("enabled")) - prev = previous or {} - out = {"cron": cron, "enabled": enabled} - if enabled: - out["enabledAt"] = (prev.get("enabledAt") if prev.get("enabled") else None) or _iso() - return out - - -# ── WAVE 23 · C5 — the ENDING, and the cycle counter that makes a loop countable. ───────────── -# The owner's words: "we can make this automation a loop, so the ending should always be defined, -# either it ends somewhere deterministic like Closed/Failed, or it goes to reset automatically, -# or the user have to click a button to reset, or after a certain amount of time it can -# automatically reset to first cycle." -# -# ⛔ `terminal` IS THE DEFAULT and every existing automation gets it, because a stored definition -# that predates this field must not start moving records on its own the day the code ships. A -# loop is a thing somebody turns on. -def clean_flow(raw, previous=None, notes=None, rt=None): - """Validate the builder's ordered action list. `(flow, error)`. - - ⚠ `rt` is the tenant wall for gated action kinds (W35-T35 / C8) and is simply forwarded. - """ - raw = raw if isinstance(raw, dict) else {} - prev = previous if isinstance(previous, dict) else {} - actions, err = clean_actions( - raw.get("actions") if "actions" in raw else prev.get("actions"), notes=notes, rt=rt) - if err: - return None, err - return {"actions": actions}, None - - -#: AMENDMENT A1 — what the `ig_profile_match` kind-flip seeds as `recordsLimit` when the -#: definition carries none. -#: -#: ⚠ 25 BECAUSE THE INPUT SAYS 25 (2026-08-06). It was 10 — the size wave 21 proved live at -#: $0.15 — while `AutomationDetail` initialises its own box to 25, and the two never met: the -#: flow node read "Up to 10 profiles · about $0.025" beside a field reading 25, on a freshly -#: created automation. Two numbers for one fact, in the panel, before anybody had typed -#: anything. **Read off the screenshot; every assertion in the battery was green.** -#: -#: This is the same species as the default-versus-guard split fixed the same day: a value -#: decided in one file and a value decided in another, describing the same thing. Aligning the -#: constants closes the only window in which they can disagree — the seed — because every later -#: state comes from the stored config. -DISCOVER_SEED_RECORDS = 25 - - -def discovery_seed_spec(kind): - """-> (default table, enrich action kind) for a discovery KIND. - - ⭐⭐ WAVE 30, OWNER REPORT 2026-08-12, verbatim: *"'When a Tiktok profile fits a criteria' - should ALWAYS have a 'Create record' EXACTLY like the Instagram one … THE ONLY DIFFERENCE IS - THE COLUMNS AND DATA SCHEMA"*. They were right, and the seeding path was the one family of - sites this wave widened everywhere ELSE: `clean_definition` planted step 1 and step 2 only when - the trigger was `ig_profile_match`, so a TikTok search stored `flow.actions = []` and had - nowhere to put what it found — MEASURED live against `auto_1` (Instagram: create_record + - enrich) versus a fresh TikTok discovery (empty). - - ⭐ THIS IS A LOOKUP, NOT A SECOND SEEDER, and that is the whole point of the ruling. One - builder plants both platforms' steps; the only things that vary are the table the record lands - in and which enrich action reads it — literally "the columns and data schema". A forked - `_ensure_tt_action` would start identical and drift, which is the failure `ENRICH_KINDS` and - `DISCOVERY_KINDS` were introduced to prevent one screen up. - - ⚠ Resolved in a FUNCTION rather than a module-level dict because `DISCOVER_TABLE` is defined - far below this line; a dict here would raise at import. - """ - if kind == "discover_tiktok": - return TT_PROFILE_TABLE, "enrich_tiktok" - return DISCOVER_TABLE, "enrich_instagram" - - -def _ensure_ig_action(flow_raw, table, enrich_kind="enrich_instagram"): - """Create record is PERMANENT step 1 on a DISCOVERY trigger — it cannot be deleted or moved. - - ⚠ THE NAME IS INSTAGRAM'S AND THE BEHAVIOUR IS BOTH PLATFORMS' (wave 30). Renaming it would - churn six gate references for no behaviour change; `enrich_kind` is what makes it general, and - it defaults to Instagram's so every pre-existing caller keeps its exact meaning. - - ⭐ OWNER RULING 2026-08-06, AND IT REVERSES WAVE 24's LAW 3. That law seeded this action once, - on the edge into the trigger, and ended "deleting it is their call, not a refusal" — with a - comment warning that re-seeding on every clean would be "a control that will not take no for - an answer". The owner's answer: *"should ALWAYS have a 'Create record' Step 1, that can't be - deleted, because the nature of that automation is that it needs to first create a record in a - database somewhere from the list of profiles to fetch."* - - Which is right, and the earlier reasoning had the category wrong: a flow whose TRIGGER - produces rows has nowhere to put them until something writes them, so an Instagram search - with no Create record is not a customised automation — it is a search whose results are - discarded. That is not a preference to respect. - - ⚠ IT KEEPS THE USER'S EDITS. Presence and POSITION are guaranteed; the table it writes to and - the values it maps are theirs. An existing `create_record` further down is MOVED to the front - rather than duplicated — re-seeding a second one would quietly double every run's writes. - """ - flow = dict(flow_raw or {}) if isinstance(flow_raw, dict) else {} - actions = [a for a in (flow.get("actions") or []) if isinstance(a, dict)] - if actions and actions[0].get("kind") == "create_record": - return _pin_unique(_ensure_enrich_step( - flow_raw if isinstance(flow_raw, dict) else flow, enrich_kind)) - at = next((i for i, a in enumerate(actions) if a.get("kind") == "create_record"), -1) - if at > 0: - actions.insert(0, actions.pop(at)) - else: - # The seed must be a config `clean_actions` ACCEPTS, or the builder shows a red banner - # and no card: the wave-23 header records four kinds that shipped with illegal seeds and - # did exactly that. `create_record` needs a ut_-prefixed table and at least one value. - actions.insert(0, { - "id": "act_1", "kind": "create_record", "enabled": True, "when": None, - # `config.label` follows the `review` action's precedent in `_clean_action_config` — - # an action naming itself, inside the untyped config bag, so no shared TS type changes. - "config": {"table": str(table or DISCOVER_TABLE), - "label": "Save the profile", - # C5: the picture and the real write finally agree — see `_pin_unique`. - "uniqueOn": "handle", - "values": {"handle": "{{handle}}"}}, - }) - flow["actions"] = actions - return _pin_unique(_ensure_enrich_step(flow, enrich_kind)) - - -def _ensure_enrich_step(flow_raw, enrich_kind="enrich_instagram"): - """⭐ 2026-08-07 (owner ruling) — ENRICH IS STEP 2 ON A DISCOVERY SEARCH. - - ⚠ WAVE 30: `enrich_kind` selects the network. Instagram's is the default so every pre-existing - caller means exactly what it meant before; TikTok passes `enrich_tiktok` and gets the identical - step shape, which is the owner's *"only the columns and data schema differ"*. - - Owner: *"make it default that this enrichment action is Step 2 always, and under Config of - Step 2 … we can have a toggle on or off."* Which is the same shape as step 1's ruling and for - the same reason: a search that finds profiles and never reads them has done half a job. The - difference is the control — step 1 is permanent because a flow without it discards its - results, while this one is permanent because the TOGGLE is how you turn it off. Deleting and - disabling would be two ways to say one thing, and only one of them survives a re-save. - - ⚠ SEEDED ON, AND ON THE FREE RUNG. `tier: "anonymous"` costs nothing, so a search that gains - this step by upgrading does not quietly start spending; switching the Source to the paid - provider is an explicit choice a person makes in front of the sentence that names the cost. - ⚠ THE COOLDOWN IS SEEDED ON TOO (30 days). On a table nobody has enriched it changes nothing — - a blank `enriched_at` is never "recent" — and the moment there IS history it stops the flow - re-buying the same profile nightly. Off-by-default would make the expensive behaviour the - accident. - - ⛔ THE EXISTENCE TEST WALKS THE FORKS (`walk_actions`). A person who moved enrichment inside an - If/then branch has one; seeding a second at the top level would enrich twice and bill twice, - which is the duplication `apply_actions` already paid for once with the pinned step 1. - """ - flow = dict(flow_raw or {}) if isinstance(flow_raw, dict) else {} - actions = [a for a in (flow.get("actions") or []) if isinstance(a, dict)] - if any(a.get("kind") == enrich_kind for a in walk_actions(actions)): - return flow_raw if isinstance(flow_raw, dict) else flow - seeded = list(actions) - # Index 1 — after the pinned Create record, because there is nothing to enrich until the - # profiles have been written as records. `insert` past the end is a plain append, so a flow - # with only step 1 lands this at the end, which IS step 2. - seeded.insert(1, { - "id": "act_enrich", "kind": enrich_kind, "enabled": True, "when": None, - "config": {"postMetrics": False, "commentMetrics": False, - "dryRun": False, "maxPosts": DEFAULT_POSTS_PER_PULL, - "fromView": "", "sortField": DEFAULT_ENRICH_SORT, "sortDir": "desc", - "limit": DEFAULT_ENRICH_LIMIT, "skipRecent": True, - "skipRecentDays": DEFAULT_ENRICH_COOLDOWN_DAYS}, - }) - return {**flow, "actions": seeded} - - -#: The key the discovery writer really upserts on. Both discovery runners key candidates on -#: `candidate_key(platform, handle)`; `handle` is the half a person can see and the half this action's -#: values carry, which is why the PINNED card shows that rather than the pair — `platform` is -#: supplied by the runner, never typed by a person. -#: ⛔ DEBT D-73 (closed wave 29): this note used to name wave 22's C6 compound — the one that -#: paired the handle with its finder — as the live key. Wave 26 · R4/R5 retired it: the identity is -#: the pair above and the TENANT is the unit, while `created_by` survives as an informational -#: "Found by" stamp that no run may branch on. -#: ⚠ The retired pair is DESCRIBED here and not reproduced, deliberately: a gate asserts this note -#: names the key `_ck` actually takes, and a verbatim quotation of the wrong one reads to that gate -#: exactly like the defect. A stale comment on correct code is how the next session reintroduces a -#: bug with a rationale attached. -IG_PINNED_UNIQUE = "handle" - - -def _pin_unique(flow): - """C5: keep `uniqueOn: "handle"` on the PINNED step 1 across every save. - - ⭐ WHY IT IS RE-STAMPED RATHER THAN MERELY SEEDED. `_clean_action_config` has no `previous` — - the builder posts the whole action list on every save — so a client that omitted the key would - silently reset it to `""`, i.e. back to append. The pinned card is a PICTURE of the engine's - own upsert (`apply_actions` skips it at runtime, see the note there), so the picture claiming - "append" while the engine upserts is exactly the surface-disagrees-with-the-engine defect this - module refuses everywhere else. - - ⛔ AND IT CANNOT MINT A CONFIG THE GUARD REFUSES — HARD RULE 11, which is the whole reason - this is a function and not one line. `_clean_action_config` refuses a `uniqueOn` the action - does not write, so the stamp is applied ONLY when `handle` is among the action's own values. - A user who remaps the card to write different columns keeps their edit and still saves; the - alternative — stamping unconditionally — is a product-seeded default that its own validator - would 400, which is precisely the post-W24 hotfix this rule exists because of. - - ⚠ NON-MUTATING, and that is load-bearing rather than tidiness. `clean_definition` passes - `prev.get("flow")` here when a PATCH carries no flow of its own — that is the STORED - definition's own dict, so writing into it would edit the live store object in memory, before - (and regardless of) any commit. Every touched level is copied instead. - """ - if not isinstance(flow, dict): - return flow - acts = flow.get("actions") - if not isinstance(acts, list) or not acts or not isinstance(acts[0], dict): - return flow - first = acts[0] - if first.get("kind") != "create_record": - return flow - cfg = first.get("config") - if not isinstance(cfg, dict) or IG_PINNED_UNIQUE not in (cfg.get("values") or {}): - return flow - if str(cfg.get("uniqueOn") or "").strip(): - return flow - return {**flow, - "actions": [{**first, "config": {**cfg, "uniqueOn": IG_PINNED_UNIQUE}}] + acts[1:]} - - -#: The two steps `_ensure_ig_action` / `_ensure_enrich_step` plant, as `(kind, id)`. Named here so -#: the seeder and the un-seeder cannot disagree about what "seeded" means. -#: ⚠ WAVE 30 — `enrich_tiktok` shares `act_enrich`. The map is keyed by KIND, and only one enrich -#: step is ever seeded per flow (the network follows the trigger), so the id cannot collide. -_IG_SEED_IDS = {"create_record": "act_1", "enrich_instagram": "act_enrich", - "enrich_tiktok": "act_enrich"} - - -def _is_untouched_ig_seed(action, table, enrich_kind="enrich_instagram"): - """Is this action still EXACTLY what the Instagram trigger planted? (wave 27 item 15) - - ⛔ THE COMPARISON IS AGAINST A FRESHLY BUILT SEED, not against a list of remembered keys, and - that is the only version that stays true: `_ensure_ig_action` and `_ensure_enrich_step` build - the same dicts one screen above, so a step gaining a config key next wave gains it here too. - A remembered key list would quietly start calling every seeded action "edited". - - ⛔ AND THE FRESH SEED GOES THROUGH `clean_actions` FIRST, which cost a red check to learn. The - stored action has been cleaned — `_clean_action_config` normalises it and adds the keys the - kind declares (`profileField: ""` on an enrich step, for one) — so comparing against the RAW - dict `_ensure_enrich_step` writes reports every seeded enrich step as "edited by a person", - and the un-seeding silently never fires for it. Comparing cleaned against cleaned is the only - version where the two sides are the same kind of object. - - ⚠ `enabled` IS DELIBERATELY NOT PART OF THE TEST for the enrich step. Its ruling says the - TOGGLE is how you turn it off — so a person who switched it off has expressed an opinion about - a step they still want, and an off seed is still a seed. - """ - if not isinstance(action, dict): - return False - kind = str(action.get("kind") or "") - if _IG_SEED_IDS.get(kind) != str(action.get("id") or ""): - return False - seeded, _seed_err = clean_actions( - (_ensure_ig_action({"actions": []}, table, enrich_kind).get("actions") or [])) - fresh = {a.get("kind"): a for a in (seeded or [])} - seed = fresh.get(kind) - if not seed: - return False - cfg, seed_cfg = dict(action.get("config") or {}), dict(seed.get("config") or {}) - if kind in ENRICH_KINDS: - cfg.pop("enabled", None) - seed_cfg.pop("enabled", None) - return cfg == seed_cfg and (action.get("when") or None) is None - - -def _drop_ig_seeds(flow_raw, table, enrich_kind="enrich_instagram"): - """Strip the trigger's own seeded steps, keeping any the person has since made theirs. - - ⭐⭐ WAVE 27 ITEM 15 (owner) — *"stale pinned create_record on trigger change"*. THE COMMENT - THAT USED TO SIT AT THE KIND FLIP ARGUED THE OPPOSITE and is rewritten there; it read: *"it - does not delete the seeded actions … a create_record the person has since re-pointed at their - own table with their own values is THEIR action now"*. That reasoning is still correct and is - exactly what this function preserves — but it was applied to EVERY seeded action, including - the ones nobody had ever opened, and the result is what the owner reported: switch an - Instagram search to Manual and you are left holding a "Save the profile" step writing - `{{handle}}` into `ut_ig_candidates` on a flow that no longer produces handles. That is not - somebody's work being protected; it is the machine's own leftover, pointed at a table the - automation has nothing to do with any more. - - ⛔ SO THE TEST IS "DID ANYBODY TOUCH IT", NOT "WAS IT SEEDED" — the same guard shape as - `_vestigial_name_field` and `_retired_tracked_field`, and for the same reason: this is a - branch that destroys something, so the conservative half is the load-bearing half. - """ - flow = dict(flow_raw or {}) if isinstance(flow_raw, dict) else {} - actions = [a for a in (flow.get("actions") or []) if isinstance(a, dict)] - kept = [a for a in actions if not _is_untouched_ig_seed(a, table, enrich_kind)] - if len(kept) == len(actions): - return flow_raw - return {**flow, "actions": kept} - - -def ig_action_pinned(defn, index): - """Is this action the one that cannot be removed? Derived, never stored — a stored `pinned` - flag is a second copy of the rule, and the copy is what a hand-written PATCH omits.""" - # ⭐ WAVE 30 — BOTH discovery kinds. It read `== "discover_instagram"`, so TikTok's step 1 (once - # seeded) would have rendered as an ordinary draggable, deletable card: the same rule the owner - # asked for "EXACTLY", applied to only one network. - return (defn or {}).get("kind") in DISCOVERY_KINDS and index == 0 - - -def clean_definition(raw, previous=None, username="", notes=None, rt=None): - """Whole-definition validation. Returns `(defn, error)`.""" - raw = raw if isinstance(raw, dict) else {} - prev = previous or {} - # ⭐ WAVE 24 · C-TRIG — THE TRIGGER IS RESOLVED FIRST NOW, because law 1 makes it the thing - # that CHOOSES the kind. Reading `raw["trigger"]["key"]` here instead would be a second - # reader of the trigger vocabulary, free to disagree with `clean_trigger` about which keys - # exist and which are refused. Gate-visible consequence, recorded in AMENDMENT A1: a payload - # invalid in BOTH its trigger and its config now answers with the trigger's sentence. - trigger, terr = clean_trigger(raw.get("trigger") if "trigger" in raw - else prev.get("trigger"), prev.get("trigger")) - if terr: - return None, terr - # C-TRIG law 2 (wave 24): a definition that names no kind is a `plain` one. Before R6 this - # refused with "unknown automation kind ''" — correct while a wizard always sent one, and a - # dead end the moment the wizard was deleted and the client's create body became {name}. - kind = _s(raw.get("kind") or prev.get("kind"), 40) or DEFAULT_KIND - drop_ig_seeds = False - if (trigger or {}).get("key") == "ig_profile_match": - # LAW 1: the trigger IS how `discover_instagram` gets chosen now. Unconditional rather - # than "when the kind is unset" — a definition whose trigger says Instagram-discovery and - # whose kind says something else is not a preference to respect, it is two halves of one - # answer disagreeing, and the trigger is the half the person actually picked. - kind = "discover_instagram" - elif (trigger or {}).get("key") == "tiktok_profile_match": - # ⭐ WAVE 29 (D-9 / R1) — LAW 1, TIKTOK'S HALF. Same rule, same reason: the trigger is how - # `discover_tiktok` gets chosen, and it is unconditional for the same reason the Instagram - # arm is — a definition whose trigger says TikTok discovery and whose kind says something - # else is two halves of one answer disagreeing. - kind = "discover_tiktok" - elif (kind == "discover_tiktok" - and ((prev.get("trigger") or {}) or {}).get("key") == "tiktok_profile_match" - and (trigger or {}).get("key") != "tiktok_profile_match"): - # ⛔⛔ LAW 1'S INVERSE, AND ITS ABSENCE ON THE INSTAGRAM SIDE WAS A MONEY BUG (see the arm - # below): with nothing to flip the kind BACK, switching the trigger to Manual left - # `RUNNERS[kind]` pointing at the discovery runner, so **Run now fired a PAID corpus - # search on a flow the person had just made manual.** Shipped here in the same change as - # law 1 rather than discovered the same way twice. - # ⚠ THE TEST IS THAT THE TRIGGER **MOVED** — comparing the RESOLVED key against the - # PREVIOUS one. Asking `"trigger" in raw` would fire on every empty patch, because - # `patch()` builds its raw as `dict(prev)` plus the caller's keys. - kind = DEFAULT_KIND - # ⭐ WAVE 30 — un-seed on the way out, exactly as the Instagram arm below does. This line - # was ABSENT and harmless for as long as TikTok had no seeds to leave behind; the moment - # the seeder above learned TikTok, its absence became wave-27 item 15's defect on the other - # network — a flow switched to Manual still carrying a "Save the profile" step nobody - # planted deliberately. Found by widening the seeder and asking what else assumed only - # Instagram could have seeds. - drop_ig_seeds = True - elif (kind == "discover_instagram" - and ((prev.get("trigger") or {}) or {}).get("key") == "ig_profile_match" - and (trigger or {}).get("key") != "ig_profile_match"): - # ⭐⭐ 2026-08-07 (owner report) — LAW 1 HAS AN INVERSE, AND ITS ABSENCE WAS A MONEY BUG. - # - # Law 1 above flips the kind TO `discover_instagram` when the trigger says Instagram - # discovery. Nothing flipped it BACK. So `kind` was inherited from `prev` forever: switch - # the trigger to Manual and the automation stayed `discover_instagram`, which means - # - # · `RUNNERS[kind]` is still `run_discover_instagram`, so pressing **Run now** on a flow - # the person had just made MANUAL fired a PAID Bright Data corpus search; - # · `ig_action_pinned` keys on the kind, so the seeded "Create record" stayed - # UNDELETABLE — the owner's report, verbatim: *"When a trigger change from the - # instagram trigger, the 'always first' create record just stuck there"*. The server - # would have accepted the delete; the CLIENT hid the control, because both halves ask - # the kind and the kind was lying. - # - # ⛔ THE TEST IS THAT THE TRIGGER **MOVED**, and the first version of this got it wrong in - # a way worth recording. It asked `"trigger" in raw` — but `patch()` builds its raw as - # `merged = dict(prev)` plus the caller's keys, so **`"trigger"` is present on EVERY - # patch**, and an empty `patch(rt, id, {})` flipped a discovery automation to `plain`. - # That turned five `section_bd` checks red and crashed the suite on a `StopIteration` - # three sections later. Comparing the RESOLVED key against the PREVIOUS one is the honest - # question: did this automation stop being an Instagram search? - # - # ⚠ NARROW ON PURPOSE, twice over. It fires only when the automation WAS on the Instagram - # trigger — a `discover_instagram` created with an explicit kind and some other trigger is - # somebody's deliberate state, not a mistake to correct. And only FROM - # `discover_instagram`: `scrape_db` and `field_instagram` are surviving kinds whose - # triggers are their own business, and clobbering them would retire them by accident. - # - # ⭐⭐ WAVE 27 ITEM 15 — IT NOW DROPS THE SEEDS NOBODY TOUCHED, and the paragraph this - # replaces argued the other way, so here is why it was half right. It read: *"it does not - # delete the seeded actions … a create_record the person has since re-pointed at their own - # table with their own values is THEIR action now, and quietly destroying it is the silent - # data loss this module refuses everywhere else."* Every word of that is still true and is - # exactly what `_is_untouched_ig_seed` protects. What it got wrong was applying the - # protection to actions NOBODY HAD EVER OPENED: the owner's report is a flow switched to - # Manual still carrying a "Save the profile" step writing `{{handle}}` into - # `ut_ig_candidates` — the machine's own leftover on a flow that no longer produces - # handles, unpinned but still there, still runnable, and still pointed at a table that has - # nothing to do with this automation. - # ⚠ APPLIED BELOW, at the flow, not here: `flow_raw` is not resolved yet at this line. - kind = DEFAULT_KIND - drop_ig_seeds = True - if kind not in KINDS: - return None, f"unknown automation kind {kind!r}" - # ⛔ DEBT D-65 — THE REFUSAL THAT BELONGS HERE IS NOT SHIPPED, AND THAT IS A DECISION. - # D-65 offers two exits for `field_instagram`: convert a surviving definition to a `plain` - # flow carrying `enrich_instagram`, or REFUSE it here with a sentence naming the replacement. - # The refusal was built and then REVERTED, because W24/R6 deliberately made `patch()` the way - # a retired kind legally comes into existence — `create` refuses, `patch` accepts, and the two - # live automations save through this function every time their owner edits them. The gate's - # own fixture helper says so in the strongest terms available: *"If R6's refusal ever moved - # from `create` into `clean_definition` (the tempting simplification), this helper would go - # red across a dozen sections, which is the alarm that change deserves."* It did, and the - # alarm worked. - # ⚠ WHAT SHIPPED INSTEAD is the half that costs nothing and was the actual complaint: every - # door that DOES refuse now names the replacement (`RETIRED_KIND_REPLACEMENT`), so nobody - # meets "unknown automation kind" for a decision made on purpose. The rest of D-65 is an - # owner-visible behaviour change — either new `field_instagram` mints stop working, or stored - # ones are rewritten under their owner — and that is a ruling, not a refactor. - name = " ".join(_s(raw.get("name") or prev.get("name") or "", MAX_NAME).split()) - if not name: - return None, "name the automation" - cfg_raw = raw.get("config") if isinstance(raw.get("config"), dict) else prev.get("config") - if kind in DISCOVERY_KINDS and prev.get("kind") != kind: - # ⭐⭐ WAVE 30 · T04 — WIDENED FROM `discover_instagram` TO BOTH DISCOVERY KINDS, AND THIS - # ONE LINE WAS THE WHOLE OF THE OWNER'S ITEM 3. Picking "When a TikTok profile fits a - # criteria" 400'd on the very first Save with *"say how many profiles to fetch — an - # UNBOUNDED discovery query is the one shape the vendor refuses outright - # (NOT_ENOUGH_FUNDS)"*, and the trigger was therefore never STORED, which is what the - # owner saw as "the Trigger won't load". - # ⛔ INSTAGRAM WAS NEVER SURVIVING ON ITS OWN MERITS: `AutomationDetail.buildConfig` sends - # no `recordsLimit` for EITHER platform on a fresh automation (its `kind` is still `plain` - # at that moment, so it falls through to `return { targetTable }`). IG worked only because - # this seed caught it. So the defect was never "TikTok is missing something Instagram has" - # — it was one hard-coded string in the single line that rescues both. - # ⚠ `prev.get("kind") != kind` is the EXACT generalisation of the old - # `prev.get("kind") != "discover_instagram"`, not a loosening: it still fires only on a - # kind FLIP, so a later save that carries a real limit is not re-seeded (asserted). - # AMENDMENT A1: the kind FLIP seeds the one field `clean_config` insists on, or the very - # first Save after picking this trigger 400s — against the stored-inert-with- - # `configured:false` pattern the whole picker is built on (see `clean_trigger`'s A3 note). - # ⛔ THE REFUSAL ITSELF IS UNTOUCHED: an explicit 0 still gets its sentence. That guard - # exists because an unbounded query is the one shape the vendor refuses outright, and - # coercing a blank into a number would be exactly the silent widening it protects against. - # Seeding cannot spend — `clean_schedule` defaults `enabled` False, so nothing runs until - # a person presses Run now or arms the schedule. - cfg_raw = dict(cfg_raw or {}) - if not _ig_int(cfg_raw.get("recordsLimit")): - cfg_raw["recordsLimit"] = DISCOVER_SEED_RECORDS - config, err = clean_config(kind, cfg_raw, prev.get("config")) - if err: - return None, err - if trigger: - # A2: re-ask completeness now that the config is validated — `ig_profile_match`'s own - # configuration IS the config, and `clean_trigger` could not see it. - trigger["configured"] = _trigger_configured(trigger, config) - try: - schedule = clean_schedule( - raw.get("schedule") if isinstance(raw.get("schedule"), dict) - else prev.get("schedule"), prev.get("schedule")) - except ValueError as e: - return None, str(e) - # (`clean_trigger` ran at the top — law 1 needs the trigger before the kind.) - flow_raw = raw.get("flow") if "flow" in raw else prev.get("flow") - # ⭐ EVERY CLEAN, NOT ONLY THE EDGE (owner ruling 6 — see `_ensure_ig_action`). The edge-only - # call was what made the action deletable; running it on every save is what makes step 1 - # permanent, and it is deliberate rather than a widened condition nobody noticed. - # ⭐⭐ WAVE 30 (owner, 2026-08-12) — BOTH DISCOVERY TRIGGERS SEED, not just Instagram's. This - # single `==` was the whole of *"why is it not copied EXACTLY"*: a TikTok search stored an - # EMPTY flow, so the product's own rule — a search that finds profiles must have somewhere to - # put them — held for one network and not the other. - if (trigger or {}).get("key") in DISCOVERY_TRIGGER_KIND: - _seed_table, _seed_enrich = discovery_seed_spec(kind) - flow_raw = _ensure_ig_action( - flow_raw, config.get("targetTable") or _seed_table, _seed_enrich) - elif drop_ig_seeds: - # ITEM 15: the trigger just STOPPED being a discovery one. Un-seed what the trigger - # planted and nobody has since made theirs — measured against the table the seeds were - # planted for, which is the PREVIOUS config's, not the one being saved. - # ⚠ And against the PREVIOUS kind's enrich action, for the same reason: the seeds to strip - # are TikTok's if that is what was planted, and asking "is this Instagram's seed?" of a - # TikTok flow answers no and silently leaves the leftover behind. - _prev_table, _prev_enrich = discovery_seed_spec(prev.get("kind")) - flow_raw = _drop_ig_seeds( - flow_raw, (prev.get("config") or {}).get("targetTable") or _prev_table, _prev_enrich) - flow, ferr = clean_flow(flow_raw, prev.get("flow"), notes=notes, rt=rt) - if ferr: - return None, ferr - # ⭐ WAVE 30 — widened with the seeder above: TikTok now HAS a pinned step 1, so the rule that - # its `config.table` is authoritative has to reach it, or the two fields that name one database - # would disagree on exactly the network that just gained the card. - if (trigger or {}).get("key") in DISCOVERY_TRIGGER_KIND: - # ⭐ WAVE 25 · R2 — THE CREATE RECORD ACTION'S `config.table` IS AUTHORITATIVE, and - # `config.targetTable` FOLLOWS IT. Two fields have been naming the same database since - # wave 24: the pinned step 1 says where profiles are written, and the config says where - # the automation's records live — and for a discovery flow those are the same database by - # construction. When they disagreed, every surface picked a different winner (the canvas - # Write node read the config, the actual write read the action), which is a bug you can - # only see by comparing two panels. - # - # ⛔ THE ACTION WINS BECAUSE IT IS THE ONE A PERSON EDITS. R2b puts the preset list on the - # action's own configuration, so the table named there is the one they were looking at. - # - # ⚠ THIS IS ALSO THE MIGRATION, and it is a migration in the shape laws 4/5 established: - # a stored definition carrying two different values is reconciled on its next clean, - # silently and exactly once, because nothing writes the losing value back. A refusal here - # would 400 every Save of a definition that was legal yesterday. - # ⭐⭐ WAVE 27 ITEM 11 — WHICHEVER SIDE MOVED WINS, and R2's "the action wins" is now the - # TIE-BREAK rather than the whole rule. The owner's report: change the database on the - # TRIGGER, save, and the picker springs back to the old one with no message. The cause is - # the branch below read unconditionally — the pinned action still named yesterday's table, - # so it overwrote a change the person had just made in front of it, silently, every time. - # - # ⛔ R2 IS NOT REVERSED, and the distinction is which QUESTION the code asks. R2 settled - # "when the two disagree, whose value is real?" — the action's, because R2b puts the - # preset list on the action's own panel, so that is the one they were looking at. That - # answers a definition at REST. A save is different: it carries an INTENT, and the honest - # question is which side this particular save changed. So: - # · the trigger-side picker moved -> it wins, and the pinned action is re-pointed to - # follow it (leaving the action behind would just re-revert on the next save); - # · only the action moved, or neither did and they were already inconsistent -> R2's - # migration, unchanged, reconciling a stored definition exactly once. - # · BOTH moved in one payload -> the action still wins. That is R2 literally, and it is - # also the only reading that cannot lose a value: the action's table is the one with - # the field mapping attached to it. - first = (flow.get("actions") or [{}])[0] - pinned = str((first.get("config") or {}).get("table") or "") \ - if first.get("kind") == "create_record" else "" - prev_target = str((prev.get("config") or {}).get("targetTable") or "") - prev_pinned = str(((((prev.get("flow") or {}).get("actions") or [{}])[0] - ).get("config") or {}).get("table") or "") - target_moved = bool(prev_target) and config.get("targetTable") != prev_target - action_moved = bool(prev_pinned) and pinned != prev_pinned - if pinned and target_moved and not action_moved: - # The person changed the trigger's database. Carry it INTO the pinned step so the two - # agree, instead of throwing their edit away to keep a stale copy consistent. - cfg1 = dict(first.get("config") or {}) - cfg1["table"] = str(config.get("targetTable") or "") - flow["actions"] = [{**first, "config": cfg1}, *(flow.get("actions") or [])[1:]] - elif pinned and pinned != config.get("targetTable"): - config["targetTable"] = pinned - # The stored label described a DIFFERENT database, so keeping it would caption the new - # one with the old one's name. Blank falls back to the key everywhere, and `ut_ensure` - # never relabels a table that already exists — so an adopted hand-made database keeps - # the name its owner gave it. - config["targetLabel"] = "" - return { - "id": _s(prev.get("id") or raw.get("id"), 40), - "name": name, "kind": kind, "config": config, "schedule": schedule, - "trigger": trigger, - # The Builder's ordered actions. An absent flow is simply an automation with no actions. - "flow": flow, - "status": prev.get("status") or {"state": "idle", "lastRunAt": "", "lastSummary": ""}, - "runs": list(prev.get("runs") or [])[:MAX_RUNS], - # ⚠ ENGINE-OWNED CONTINUATION STATE, and it is NOT config. A discovery snapshot takes ~20 - # minutes to build, so a run persists its id here and the NEXT run collects it. It is - # carried through `patch` untouched because a user pressing Save must not silently orphan - # a set the vendor is already building (and already counting against the funds gate). - "state": dict(prev.get("state") or {}), - "created": prev.get("created") or _iso(), - "createdBy": prev.get("createdBy") or username, - # ⭐⭐ WAVE 35 · T37 — `system` IS STICKY ACROSS A PATCH, AND IT IS **NEVER READ FROM `raw`**. - # - # ⛔ THE BUG THIS CLOSES WAS ALREADY WRITTEN AND WOULD HAVE SHIPPED. `routes_automation. - # delete_automation` refuses any stored row carrying `system`, which is what makes a seeded - # system agent undeletable — and this function is a WHITELIST that did not carry the key, so - # the FIRST EDIT of such an agent silently stripped its marker and made it deletable. The - # seed would then mint it again on the next list call: a delete that appears to work and - # undoes itself. Found by asserting the round trip rather than by reading the code. - # - # ⛔ FROM `prev` ONLY. Reading it from `raw` would let ANY caller POST - # `{"system": "anything"}` and mint themselves an automation nobody can delete — a - # privilege escalation through a field nobody validates. The flag is granted by the seeder - # and inherited from the stored row, never asserted by a payload. - # ⚠ Same shape and same reason as the pre-set FIELD flag, which DESIGN.md §4 already - # describes as "sticky across a PATCH". One idea, one behaviour, two objects. - **({"system": _s(prev.get("system"), 40)} if prev.get("system") else {}), - }, None - - -def set_state(rt, auto_id, patch_state): - """Merge into ONE automation's engine state. A key set to None is removed. - - Its own tiny store write rather than a field on `_commit_run`, because the handoff must - survive a run that ends in `error` — a snapshot the vendor is building does not stop existing - because the run that started it failed afterwards. - """ - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - d = cur.get(str(auto_id)) - if d is None: - return cur - st = dict(d.get("state") or {}) - for k, v in (patch_state or {}).items(): - st.pop(k, None) if v is None else st.__setitem__(k, v) - d["state"] = st - return cur - - # ⚠ W31-T35 — THE ONE `keeps_defs=True` IN THIS MODULE. `set_state` writes `state` and never a - # trigger, and `grid_hook` calls it once per CREATED RECORD; clearing the definitions memo here - # would re-read the whole bucket one row later than before and undo D-134 entirely. Safe - # because `grid_hook` reads exactly one state key (`rcHighwater`) and mirrors its own write - # into the memo in the same statement. Read `_store_update`'s note before adding a second. - _store_update(rt, _up, flush="sync", keeps_defs=True) - - -def _without_retired_board(definition): - """Return a definition with retired Board-only state removed, without mutating the store. - - This protects scheduled/manual execution before the next UI list request persists the - migration. It removes only the former Board configuration, actions, and audit trail; normal - actions and all user data remain intact. - """ - if not isinstance(definition, dict): - return definition, False - out, changed = dict(definition), False - cfg = out.get("config") - if isinstance(cfg, dict) and "lanes" in cfg: - cfg = dict(cfg) - cfg.pop("lanes", None) - out["config"] = cfg - changed = True - - def _actions(actions): - nonlocal changed - clean = [] - for action in actions if isinstance(actions, list) else []: - if not isinstance(action, dict): - clean.append(action) - continue - if action.get("kind") == "review": - changed = True - continue - item = dict(action) - if item.get("kind") == "group" and isinstance(item.get("config"), dict): - group_cfg = dict(item["config"]) - branches = group_cfg.get("branches") - if isinstance(branches, list): - kept = [] - for branch in branches: - if not isinstance(branch, dict): - changed = True - continue - next_actions = _actions(branch.get("actions")) - if not next_actions: - changed = True - continue - kept.append({**branch, "actions": next_actions}) - if not kept: - changed = True - continue - if kept != branches: - changed = True - group_cfg["branches"] = kept - item["config"] = group_cfg - elif isinstance(group_cfg.get("actions"), list): - nested = _actions(group_cfg["actions"]) - if not nested: - changed = True - continue - if nested != group_cfg["actions"]: - group_cfg["actions"] = nested - item["config"] = group_cfg - clean.append(item) - return clean - - flow = out.get("flow") - if isinstance(flow, dict): - next_flow = dict(flow) - actions = _actions(flow.get("actions")) - if actions != flow.get("actions"): - next_flow["actions"] = actions - if "ending" in next_flow: - next_flow.pop("ending", None) - changed = True - out["flow"] = next_flow - if "reviews" in out: - out.pop("reviews", None) - changed = True - return out, changed - - -def retire_automation_board_state(rt, tables=None): - """Persist the idempotent Board retirement, then remove its generated table fields. - - ``tables`` is the optional lent snapshot of the `user_tables` bucket (W29-T01) — it reaches - the stage scan and nothing else. - """ - fields = retire_automation_stage_fields(rt, tables=tables) - try: - stored = dict(rt.get(STORE_KEY) or {}) - except Exception: - return {"definitions": 0, **fields} - plan = {str(aid): cleaned for aid, raw in stored.items() - for cleaned, changed in [_without_retired_board(raw)] if changed} - if not plan: - return {"definitions": 0, **fields} - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - for aid, cleaned in plan.items(): - if aid in cur: - cur[aid] = cleaned - return cur - - _store_update(rt, _up, flush="sync") - return {"definitions": len(plan), **fields} - - -#: ⭐⭐ WAVE 31 · T35 (D-134) — the definitions memo, and the two things that make it safe. -#: `tenant -> (monotonic_at, defs)`. Read ONLY through `all_definitions(..., cached=True)`, which -#: exactly one caller uses (`grid_hook`). -_DEFS_MEMO = {} -#: Deliberately SHORT. This exists to collapse one burst of row events into one read, not to be a -#: cache — an import of 20,000 rows arrives in far less than this, and a stale trigger for two -#: seconds is bounded and recoverable where a stale one for a minute is a mystery. -_DEFS_TTL = 2.0 - - -def _store_update(rt, fn, flush="sync", keeps_defs=False): - """THE one door every write to the automations bucket goes through — and the ONLY reason it - exists is that the memo's invalidation must be DERIVED rather than remembered. - - ⚠ `keeps_defs=True` IS AN EXPLICIT, SINGLE-SITE OPT-OUT, and it is here because the safe - default would otherwise make the memo useless on the exact path it was built for. `set_state` - writes `state` and NEVER a trigger, and it is called by `grid_hook` itself once per created - record — so clearing on it would mean a 20,000-row import re-read the bucket 20,000 times - anyway, one row later than before. The opt-out is safe because `grid_hook` reads exactly one - state key (`rcHighwater`) and mirrors its own write into the memo in the same statement. - ⛔ The DEFAULT is the safe one, so a thirteenth writer added next wave gets invalidation - without knowing this exists; only a caller that has read this paragraph can opt out. - - ⛔ THE ALTERNATIVE WAS TWELVE CALL SITES. `rt.update(STORE_KEY, …)` appeared twelve times in - this module; hanging an invalidation on each would be a hand-maintained list, and the way that - fails is silent: a thirteenth writer added next wave leaves `grid_hook` firing triggers off a - definition set that no longer exists, with nothing red. Same defect class as `patch`'s - hand-maintained patchable-key list, which this module already documents as *"the silent-drop - seat"* [[a-constant-two-features-share]]. - - ⚠ IT CLEARS THE WHOLE MEMO, not this tenant's row, on purpose: identifying the tenant here - would mean trusting `getattr(rt, 'key')` at a WRITE, and a wrong answer there is a stale - trigger set for another tenant. The memo holds at most a handful of entries and the correct, - boring thing costs nothing. - """ - if not keeps_defs: - _DEFS_MEMO.clear() - return rt.update(STORE_KEY, fn, flush=flush) - - -def all_definitions(rt, cached=False): - """Every automation definition for this tenant. - - ⭐⭐ `cached=True` IS D-134's FIX, AND IT IS OPT-IN FOR A REASON. `grid_hook` runs ONCE PER ROW - EVENT and called this unconditionally, so a 20,000-row import performed 20,000 whole-document - deep copies of the automations bucket — `Store.get` re-serialises on every call, hit or miss, - under the store lock. Every other caller is a request handler that reads it once, so widening - the memo to all of them would trade a real guarantee (a request sees the current store) for - nothing measurable. - - ⛔ THE MEMO IS THE SAME OBJECT ON A HIT, and `grid_hook` MUTATES it deliberately — see the - `rcHighwater` write there. That is not a leak of an implementation detail, it is the fix to the - hazard a naive memo creates: `grid_hook` both READS `state.rcHighwater` and WRITES it through - `set_state`, so a memo that went stale against its own write would re-fire `record_created` - for records it had already handled, breaking A2(4)'s *"once per record EVER — undo-proof"*. - Writes through `_store_update` drop the memo; the one write `grid_hook` makes to its OWN copy - is mirrored into it in the same statement. - """ - if cached: - key = str(getattr(rt, "key", "") or "") - hit = _DEFS_MEMO.get(key) - if hit is not None and (time.monotonic() - hit[0]) < _DEFS_TTL: - return hit[1] - try: - raw = dict(rt.get(STORE_KEY) or {}) - except Exception: - return {} - out = {str(aid): _without_retired_board(defn)[0] for aid, defn in raw.items()} - if cached: - _DEFS_MEMO[str(getattr(rt, "key", "") or "")] = (time.monotonic(), out) - return out - - -def _new_id(existing): - n = 1 - while f"auto_{n}" in existing: - n += 1 - return f"auto_{n}" - - -#: C2's three answers to "which database does this automation work on?" — the FIRST question the -#: create wizard asks (owner item 3: the kind is not the first thing a person picks, the data is). -TARGET_MODES = ("existing", "new", "automated") - - -def resolve_target(rt, raw, username=""): - """C2: turn the wizard's `target` into a bound table key. Returns `(config_patch, error)`. - - - `existing` — bind a database that is already there. - - `new` — mint a blank one now, so the automation has somewhere to write before its first - run instead of conjuring a table the person never agreed to. - - `automated` — leave it to the runner: a scraping automation MINTS its target on first run - (`ut_ensure`), stamped `source: "Automation"`, which is what makes it a machine database. - Nothing is created here, deliberately — an empty table created up front for a scrape that - never runs is litter nobody can explain. - """ - if not isinstance(raw, dict) or not raw: - return {}, None # no target block = the pre-wizard shape - mode = _s(raw.get("mode"), 20).strip() or "existing" - if mode not in TARGET_MODES: - return None, f"{mode!r} is not one of: " + ", ".join(TARGET_MODES) - if mode == "automated": - label = " ".join(_s(raw.get("label"), 60).split()) - return ({"targetLabel": label} if label else {}), None - if mode == "existing": - key = _s(raw.get("table"), 60).strip() - if not key: - return None, "choose the database this automation works on" - t = ut_get(rt, key) - if t is None: - return None, f"{key!r} is not a database in this workspace" - return {"targetTable": key, "targetLabel": t.get("label") or key}, None - label = " ".join(_s(raw.get("label"), 60).split()) - if not label: - return None, "name the new database" - import core.user_tables as _ut_new - key = _ut_new.create(label, username or "automation", st=rt) - if not key: - return None, ("the database could not be created. This workspace may be at its table " - "limit") - return {"targetTable": key, "targetLabel": label}, None - - -def _unmint(rt, key): - """Delete a database this call minted moments ago, because the call is refusing (D-50). - - ⚠ FAIL-QUIET BY DESIGN. The caller is already returning a refusal with a sentence the user - needs to read; a rollback that raised would replace that sentence with a 500 and lose the - reason the create failed in the first place. The worst case of a swallowed failure here is - the orphan we had before — strictly no worse, and the refusal still reaches the person. - """ - if not key: - return - try: - import core.user_tables as _ut_del - _ut_del.delete(key, st=rt) - except Exception: # noqa: BLE001 - pass - - -def create(rt, raw, username="", notes=None): - existing = all_definitions(rt) - if len(existing) >= MAX_AUTOMATIONS: - return None, f"this workspace is at the {MAX_AUTOMATIONS}-automation limit" - raw = dict(raw or {}) - # ⭐ WAVE 24 — DEBT D-50. `resolve_target` MINTS a database for `mode:"new"`, and validation - # happens after it, so a correct refusal ("a source URL is required") used to leave an - # orphaned `ut_*` table behind with no automation pointing at it. MEASURED at the W23 close: - # three refused creates, two orphaned databases, deleted by hand. A person mis-filling a form - # twice silently accumulated empty databases in their nav. - # - # ⛔ THE ROLLBACK IS DERIVED, NOT DECLARED, and that is the point of doing it this way. It - # does not test for `mode == "new"`; it asks "did the table this call's target resolved to - # exist BEFORE this call?" So it closes the hole for any future target mode that mints, and - # a fifth mode cannot reopen it by forgetting a branch. Scoped to THIS call's own target - # rather than "any table that appeared", so a concurrent create in another request is never - # collateral. - before = set(ut_all(rt) or {}) - patch_cfg, terr = resolve_target(rt, raw.pop("target", None), username) - if terr: - return None, terr - minted = str((patch_cfg or {}).get("targetTable") or "") - minted = minted if minted and minted not in before else "" - if patch_cfg: - raw["config"] = {**(raw.get("config") or {}), **patch_cfg} - # 2026-08-10 — point a targetless discovery flow at the profile database this tenant ALREADY - # has, instead of letting the pure validator mint a second empty one. See - # `discover_default_table`; a target the caller named is never rewritten. - raw = _apply_discover_default(rt, raw) - # W35-T35 (C8): `rt=` is the tenant wall for gated action kinds. Both doors pass it; a door - # that forgot would REFUSE the kind, not admit it (`TENANT_GATED_ACTIONS` is fail-closed). - defn, err = clean_definition(raw, None, username, notes=notes, rt=rt) - if err: - _unmint(rt, minted) - return None, err - # ⭐ WAVE 24 · OWNER RULING R6 — "scrape_db and field_instagram survive on the automations - # that already use them and NO NEW ONE CAN BE CREATED." - # - # ⛔ HERE, IN `create`, AND NEVER IN `clean_definition`. A PATCH of one of the two live - # automations runs through `clean_definition` with `prev["kind"]` already set, so refusing - # the kind there would make both of them unsaveable — the ruling retires the door, not the - # automations behind it. Enforced on the server rather than left to the deleted wizard: a - # wall that holds only because the client stopped asking is a convention, not a wall. - if defn["kind"] in RETIRED_KINDS: - _unmint(rt, minted) - return None, (f"{KIND_LABELS.get(defn['kind'], defn['kind'])!r} automations are no " - f"longer created. The ones already using it keep working. " - f"{RETIRED_KIND_REPLACEMENT.get(defn['kind'], '')}") - defn["id"] = _new_id(existing) - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - cur[defn["id"]] = defn - return cur - - # `async` for the reason spelled out at `remove` and applied at `patch` — a create is the same - # blocking commit inside the same request, and the person is waiting on it in the same way. - # ⚠ NO PRESET GUARD HERE, deliberately: `patch` may skip the spawn because it can compare - # against a PREVIOUS definition, and a create has none — R10's columns must exist before a run, - # so this call stays unconditional. - _store_update(rt, _up, flush="async") - _presets_after_write(rt, defn, username) # R10: the columns exist before a run - if defn.get("trigger"): - _seed_event_state(rt, defn) - return defn, None - - -def patch(rt, auto_id, raw, username="", notes=None): - existing = all_definitions(rt) - prev = existing.get(str(auto_id)) - if prev is None: - return None, "no such automation" - merged = dict(prev) - # ⚠ A HAND-MAINTAINED PATCHABLE-KEY LIST, and it is the silent-drop seat of this module: a - # key missing here is accepted by the route, validated by `clean_definition`, and then - # DISCARDED — the client shows a saved flow that the store never received, and nothing goes - # red. `flow` joined it in the same edit that created `flow` (wave 23), which is the only - # ordering that never has a window where the bug exists. - for k in ("name", "config", "schedule", "kind", "trigger", "flow"): - if k in (raw or {}): - merged[k] = raw[k] - # 2026-08-10 — the PATCH door needs it too, and this is the one that actually bit: picking the - # Instagram trigger on an existing automation flips the kind, and the very next Save runs a - # config that has never carried a target through the pure validator, which mints the empty - # `ut_ig_candidates`. `create` alone would have left the commonest path untouched. - merged = _apply_discover_default(rt, merged) - defn, err = clean_definition(merged, prev, username, notes=notes, rt=rt) - if err: - return None, err - defn["id"] = str(auto_id) - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - cur[defn["id"]] = defn - return cur - - # ⛔ WHY THIS IS `async` — owner report 2026-08-12 (wave 30), MEASURED on the deployed build. - # Owner, verbatim: *"debug Automation module, its extremely slow, even renaming an Automation - # takes forever"*. A rename paid a BLOCKING HF commit inside the request, and `Store.update`'s - # sync branch holds `self._lock` across a strict DOWNLOAD **and** the upload — the same lock - # `Store.get` takes — so one rename stalls every OTHER reader of that tenant's store for its - # whole duration. That is the "module is slow" half a per-route fix never reaches. - # The reasoning, the read-your-writes contract and the `user_tables.add_row` precedent are - # written out in full at `remove` above (same wave, same owner report); this is that ruling - # applied to the save path rather than a second argument for it. - _store_update(rt, _up, flush="async") - # ⛔ THE PRESET SPAWN RUNS ONLY WHEN ITS OWN INPUTS MOVED — the other half of the same report. - # MEASURED as `nurilab-admin` on `4258a93`: **7,912 ms** for one rename against a **288 ms** - # warm `GET /automations` on the same connection, i.e. 27x the read this wave just made fast - # (T12/T13). A rename changes `name`; `_presets_after_write` reads `config`, `flow` and - # `trigger` and NOTHING else, then pays a full `user_tables` read (`ut_get`/`ut_ensure`; - # `rt.get` deep-copies by design, documented ceiling 35.8 MB / ~1.4 s) to re-derive columns - # that cannot have changed. ⭐ The generalisable half: T12/T13 took `ut_all` off the automation - # READ path this wave and every sibling WRITE path still carried it — a performance fix scoped - # to "the slow page" leaves the same call in every route beside it - # [[read-a-gates-predicate-for-what-it-excludes]]. - # ⛔ THE DANGEROUS DIRECTION IS SKIPPING, NEVER RUNNING, so the predicate is the INPUT SET of - # the function being skipped — not a hand-listed set of "cheap" edits, which is how a guard - # silently un-ships W25/R10 the next time the spawn learns to read a fourth key. When none of - # them moved the call is a no-op by construction (it re-derives from those same keys), and the - # run-time `ut_ensure` R10 deliberately kept is still the backstop if the TABLE drifted - # underneath — which is not a thing a rename should be repairing. Shaped after the trigger - # comparison directly below, which already compares `prev` against the cleaned definition. - # ⭐⭐ WAVE 31 · T30 — AND THE PREDICATE IS NOW THE SPAWN'S OWN ARGUMENT, which is the fix. - # The four-whole-objects test above was right in principle and inert in practice: `persist` - # sends a REBUILT `config`, so `prev["config"] != defn["config"]` on essentially every save and - # the guard fired every time — sparing only *rename*, the one edit the owner stopped - # complaining about after wave 30. `preset_inputs` is the canonical projection of exactly what - # `_spawn_presets` reads, and `_spawn_presets` is handed nothing else, so this comparison - # cannot go stale behind a future input the way a hand-listed key set would. The full argument, - # including why this is STRONGER than what it replaces rather than a relaxation, is in - # `preset_inputs`' own docstring — read that before widening or narrowing this line. - if preset_inputs(prev) != preset_inputs(defn): - _presets_after_write(rt, defn, username) # R10: also on the save that RE-TARGETS - # A2(1): a NEW or CHANGED condition trigger starts with everything currently matching - # DISARMED — enabling never fires for records already in the state. - if (defn.get("trigger") or {}) != ((prev.get("trigger")) or {}): - _seed_event_state(rt, defn) - return defn, None - - -def _presets_after_write(rt, defn, username): - """⭐ WAVE 25 · R10 — THE PRESET COLUMNS APPEAR ON SAVE, not on the first run. - - Owner ruling R10: choosing the trigger / naming the Create record target spawns the preset - columns immediately, "so R2b's list is real and the database is visible before anything is - spent". That last clause is the point — a person can look at the database, see the sixteen - columns, and decide whether to arm the schedule, instead of paying a vendor to find out what - they are agreeing to. - - ⛔ THE RUN-TIME `ut_ensure` STAYS AS THE BACKSTOP (R10 says so explicitly), and it costs - nothing to keep: `ut_ensure` skips the write entirely when no field would change, so the first - run of a saved automation finds the table already correct and spends no commit. - - ⚠ IT SPAWNS `CANDIDATE_FIELDS`, NOT `PRESET_PROFILE_FIELDS`. The discovery target needs the - search bookkeeping (`found_count`, `first_found`, `created_by`) as well as the - preset set, and spawning a subset here would mean the columns changed between Save and the - first run — two different answers to "what does this database look like", which is the exact - disagreement R10 exists to remove. - - ⚠ A WORKSPACE AT ITS TABLE CEILING SPAWNS NOTHING AND THE SAVE STILL SUCCEEDS. `ut_ensure` - returns the key without creating when `MAX_UT_TABLES` is reached; refusing to save an - automation because the workspace is full would be a worse answer than saving one whose table - arrives later. The run-time path reports that condition LOUDLY (`ut_missing`, D-11), so the - honest failure still has exactly one home. - """ - _spawn_presets(rt, preset_inputs(defn), username) - - -#: The enrich actions whose OWN config decides which preset columns and child databases a save -#: spawns, and the exact keys read off each. DECLARED here rather than spelled inside -#: `preset_inputs`, so `_spawn_presets` and the save-path guard cannot drift apart by one key. -PRESET_ACTION_INPUTS = { - "enrich_instagram": ("profileField",), - "enrich_tiktok": ("profileField", "postMetrics", "commentMetrics"), -} - - -def preset_inputs(defn): - """⭐⭐ WAVE 31 · T30 — EXACTLY the values `_spawn_presets` consumes off a definition, canonical. - - ⛔ THIS IS THE WHOLE POINT OF THE TICKET, so it is worth being precise about what changed and - why it is SAFER than what it replaces, not weaker. Owner item 7, verbatim: *"It still takes - forever to change the Config for automation as well. It just say Saving... and takes a long time - for me to change options and configs etc."* - - Wave 30 guarded the spawn on `any(prev[k] != defn[k] for k in ("config","flow","trigger", - "kind"))` and reasoned — correctly — that *"the predicate is the INPUT SET of the function being - skipped, not a hand-listed set of cheap edits"*. The flaw was not the reasoning; it was that the - input set was stated as four WHOLE OBJECTS while the spawn reads a handful of leaves out of - them. `AutomationDetail.tsx::persist` sends `config: buildConfig()` — a value RECONSTRUCTED from - React state, not the stored object — so the comparison is against something rebuilt on every - save and the guard never fires on the one edit the owner is complaining about. W30's fix spared - *rename* (the client omits `flow`/`trigger` entirely, and `kind`/`name` are unchanged), which is - exactly the edit the owner stopped complaining about after wave 30. - - ⭐ WHY NARROWING IS SAFE HERE AND WAS NOT SAFE THEN: this projection is not a hand-listed - allow-list that a future wave can silently outgrow. `_presets_after_write` hands this dict to - `_spawn_presets` and `_spawn_presets` **never receives `defn` at all** — so it is structurally - incapable of reading a fifth key that this function does not carry. Teaching the spawn to read - something new REQUIRES adding it here, and the guard then follows for free. That is a stronger - guarantee than wave 30's, which held only as long as somebody remembered the comment. - - ⚠ CANONICAL, because the input is a rebuild: each leaf is normalised exactly the way the spawn - itself normalises it (`str(...).strip()` for a field name, `bool(...)` for a switch). Without - that, `postMetrics: false` and a missing `postMetrics` would compare unequal while the spawn - treats them identically — a guard that fires on a difference its consumer cannot see is the - same defect in a smaller font. - - ⚠ IT DELIBERATELY COVERS THE DISCOVERY BRANCH **AND** THE ENRICH BRANCH IN ONE PROJECTION, even - though today's control flow returns before the second can run for a discovery automation. That - early return is D-178 (W31-T34) and it is going away; a projection built around today's branch - would silently narrow the guard the moment it does. - """ - defn = defn or {} - cfg = defn.get("config") or {} - actions = ((defn.get("flow") or {}).get("actions") or []) - return { - "target": str(_flow_table(defn) or cfg.get("targetTable") or ""), - "label": str(cfg.get("targetLabel") or ""), - "flowId": str(defn.get("id") or ""), - "discoveryKind": DISCOVERY_TRIGGER_KIND.get( - (defn.get("trigger") or {}).get("key") or ""), - "actions": { - kind: [{k: (_norm_switch(a, k) if k != "profileField" - else str((a.get("config") or {}).get(k) or "").strip()) - for k in keys} - for a in _actions_of_kind(actions, kind)] - for kind, keys in PRESET_ACTION_INPUTS.items() - }, - } - - -def _norm_switch(action, key): - """A capture switch as the spawn reads it — `bool`, so absent and False are ONE value.""" - return bool((action.get("config") or {}).get(key)) - - -def _spawn_presets(rt, inputs, username): - """The spawn itself. Takes `preset_inputs(defn)` — never the definition — see that docstring. - - ⭐⭐ WAVE 31 · T30, THE SECOND HALF: **ONE `user_tables` read for the whole pass, not five.** - Every `ut_ensure` below used to take its own copy of the tenant document through - `ut_get` → `ut_all` → `rt.get`, which deep-copies unconditionally (ceiling 35.8 MB / ~1.4 s), and - the TikTok arm reaches FOUR of them plus the `ut_get` above — measured **7,912 ms** on - `4258a93`. Lending one snapshot is the same fix W30-T13 made on the automation READ path - (`routes_automation.py::_LentTables`); this is deliberately NOT a third copy of that class, but - the `tables=` parameter `retire_automation_stage_fields` in this very module already - established, because here every consumer is a function we own and pass to directly. - """ - target = inputs["target"] - if not target: - return - tag = inputs["flowId"] - ig_actions = inputs["actions"]["enrich_instagram"] - tt_actions = inputs["actions"]["enrich_tiktok"] - try: - # ⭐ THE ONE READ. Everything below answers out of this snapshot. - tables = ut_all(rt) - # ⭐⭐ WAVE 30 · T06 — BOTH DISCOVERY KINDS SPAWN ON SAVE, and TikTok's absence here was - # R10 simply not holding for the second platform: `TT_PROFILE_FIELDS` reached `ut_ensure` - # at exactly ONE site — inside `run_discover_tiktok` — so the database did not exist until - # money had already been spent finding out what was in it. That is the precise pre-R10 - # behaviour `verify_automation`'s NC39 forbids on the Instagram side. - # ⛔⛔ THE DISCRIMINATOR IS THE **TRIGGER**, NOT THE KIND, AND THE GATE PAID FOR THAT - # SENTENCE. Widening this to `kind in DISCOVERY_KINDS` looks equivalent — law 1 makes a - # discovery trigger choose its kind — but the implication only runs ONE WAY. A definition - # may carry `kind: "discover_instagram"` with **no trigger at all**: law 1 sets the kind - # from the trigger and never the reverse, so any direct API create can do it, and one of - # this repo's own fixtures does. MEASURED: the kind test spawned a preset database under a - # dry-run fixture whose check asserts that no table exists — a red on correct-looking code, - # caught only because that check happened to exist. - # ⇒ R10 is a rule about what somebody PICKED IN THE PICKER, so it reads the picker's own - # answer. `DISCOVERY_TRIGGER_KIND` is that map, and a gate asserts it agrees with law 1. - # ⚠ TRACKED because it is the ONE key several `ut_ensure` calls in this pass can share: the - # discovery arm and both enrich arms all ensure `target`. A later one must not answer out of - # a snapshot an earlier one invalidated — see the `tables=` arguments below. - wrote_target = False - _dkind = inputs["discoveryKind"] - if _dkind: - _, _, _spawn_label, _spawn_fields = discovery_facts(_dkind) - ut_ensure(rt, inputs["label"] or _spawn_label, _spawn_fields, username, - key=target, flow_tag=tag, lock_fields=True, tables=tables) - # ⭐⭐ WAVE 31 · T34 (D-178) — AND THE `return` THAT USED TO BE HERE IS GONE. - # - # ⛔ WHAT IT COST: a DISCOVERY automation could never spawn its child databases. The - # capture switches below are the ones that promise `ut_tt_posts` / `ut_tt_comments` / - # `ut_tt_post_snapshots` at SAVE, before any money is spent (W30-T10) — and only a - # `plain` flow ever reached them, because a discovery flow left this function three - # lines earlier. So the exact automation the owner would build for TikTok discovery — - # find profiles, then capture their posts — silently got the profile table and nothing - # else, and the toggles it showed were promises the save path never kept. - # ⚠ IT IS A TRAP, NOT TODAY'S ONLY SYMPTOM: nurilab's child tables are absent for a - # CONFIGURATION reason as well, so fixing this alone will not make them appear there. - # - # ⚠ THE SNAPSHOT IS NOW STALE FOR `target`, and the arms below resolve their profile - # binding out of that very table (`bound` reads its `fields`). One re-read, paid only on - # a save that actually reached the discovery spawn — correctness before the read count, - # and it is one read against the 41 this function used to take. `wrote_target` stays - # False precisely BECAUSE we refreshed: it tracks staleness, not "did somebody write". - tables = ut_all(rt) - enrich_actions = ig_actions - table = tables.get(target) or {} - named = next((a["profileField"] for a in enrich_actions if a["profileField"]), "") - bound = next((str(f.get("key") or "") for f in table.get("fields") or [] - if isinstance(f.get("profile"), dict)), "") or named - if enrich_actions and bound: - ut_ensure(rt, table.get("label") or target, _profile_schema_for(bound), username, - key=target, flow_tag=tag, lock_fields=True, tables=tables) - wrote_target = True - # ⭐ WAVE 30 · T08 — the TikTok half of W25/R10: the columns an `enrich_tiktok` step will - # fill appear when the automation is SAVED, not when money is first spent. Resolved - # through `profile_field_key(..., source=PROFILE_SOURCE_TT)` so it cannot adopt an - # Instagram binding, and through the SAME `_tt_profile_schema_for` the runner's own top-up - # uses — R10's rule is that the column set does not change between Save and the first run, - # and one shared resolver is what makes that structural instead of a convention. - if tt_actions: - tt_named = next((a["profileField"] for a in tt_actions if a["profileField"]), "") - tt_bound = profile_field_key(table, tt_named, source=PROFILE_SOURCE_TT) - if tt_bound: - ut_ensure(rt, table.get("label") or target, _tt_profile_schema_for(tt_bound), - username, key=target, flow_tag=tag, lock_fields=True, - tables=None if wrote_target else tables) - # ⭐ WAVE 30 · T10 — and the CHILD databases the step's own switches promise, at - # SAVE, before any money. R10's rule is "the columns a step will fill appear when - # you press Save"; a `postMetrics` toggle whose database only exists after a paid - # run is the same complaint one level up. - # ⛔ GATED ON THE SWITCHES, never spawned unconditionally: a database that exists - # because somebody looked at a toggle, and then never fills, is the "SECOND, empty - # database" the discovery default was rewritten to stop producing. - for _flag, _child in (("postMetrics", TT_POSTS_TABLE), - ("postMetrics", TT_POST_SNAPSHOTS_TABLE), - ("commentMetrics", TT_COMMENTS_TABLE)): - if any(a[_flag] for a in tt_actions): - # R9 (W31-T32): the child arrives LOCKED, from its own declaration. - ut_ensure(rt, TT_TABLE_LABELS[_child], TT_TABLE_FIELDS[_child], username, - key=_child, flow_tag=tag, lock_fields=True, tables=tables, - record_mode=tt_record_mode(_child)) - except Exception as e: # noqa: BLE001 - # ⛔ NEVER FAILS THE SAVE. The definition is already committed by the time this runs, so - # raising here would answer 500 for an automation that IS stored — the caller would retry - # and create a second one. The columns then arrive on the first run, which is precisely - # the behaviour this function is an improvement on rather than a replacement for. - print(f"[aios-auto] preset spawn deferred to the first run: {type(e).__name__}: {e}") - - -def remove(rt, auto_id): - aid = str(auto_id) - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - cur.pop(aid, None) - return cur - # ⛔ WHY THIS IS `async` AND NOT `sync` (owner report 2026-08-12, wave 30, MEASURED). - # Owner, verbatim: *"it takes forever to delete an automation"*. A delete was paying up to - # TWO BLOCKING UPLOADS inside the request — this one, plus the whole-document rewrite below, - # which fires for every DISCOVERY automation because those always tag the columns they - # spawned (deleting one marked all 30 columns of `ut_tt_profile` disabled). Measured 977 ms - # for the CHEAPEST case (a `plain` flow, no tagged columns) against a 757 ms `GET /automations` - # baseline on the same connection; the discovery case is strictly worse by a full-document - # read plus a full-document commit. - # - # `flush="async"` is not a weakening: `Store.update`'s async branch applies the mutation to - # the in-process cache, marks it owned+dirty and schedules a COALESCING worker, and the - # class's read-your-writes contract means the very next `GET /automations` already sees the - # deletion. This is the same trade `user_tables.add_row` makes for ROW creation — strictly - # more valuable data than an automation definition — so a delete is not the place to be - # stricter than a row insert. [[sync-write-eats-pending-async-write]] is the hazard in the - # OTHER direction (a sync writer discarding a pending async write) and `Store.update`'s dirty - # branch already handles it. - _store_update(rt, _up, flush="async") - # C8 — the fields this flow tagged are DISABLED with the reason on them, never orphaned - # silently: the column keeps its values and its config, and says why it stopped. Scanned - # first so a delete with no tagged fields costs no store commit. - # ⚠ HONEST RESIDUAL, stated rather than smoothed: this scan still costs ONE full read of the - # tenant's `user_tables` document on EVERY delete (`rt.get` deep-copies by design), and the - # update below reads it a second time. Removing that needs a tagged-field index, which is a - # feature and not a wave-tail edit — booked rather than improvised. What it no longer costs - # is the part the owner could feel: the blocking network commits. - # ⭐ WAVE 31 · T39(a) — D-177(a): ONE WALK, AND THE PLAN CARRIES WHAT IT FOUND. - # This used to scan the whole document to answer a boolean (`hit`), then walk the WHOLE - # document again inside `_disable` to re-derive the same matches. The scan now records the - # `(table, field)` pairs it found and the mutation applies exactly those — the shape - # `bind_unbound_fields` in this module already uses, so a delete costs one walk instead of two - # and the write touches only the fields it named. - # ⭐⭐ WAVE 33 (D-179) — THE SCAN NO LONGER READS THE ROWS, and that is where the cost was. - # - # This walk asks ONE question — "which fields did this flow tag?" — and it has never looked at - # a row to answer it. It was nevertheless taking `ut_all`, a whole deep copy of the tenant's - # `user_tables` document, of which **99.9% of the bytes are `rows` nothing here reads** - # (measured on tenant #0: 28.6 MB, 81,330 rows, 703 ms warm). A's C5 projection makes that - # ~0.1% of the bytes. - # ⛔ HANDED TO A READ AND NOTHING ELSE. `lend_defs` serves `_Projected` values, so reaching - # `defn['rows']` raises a `KeyError` NAMING the projection — by design — and `plan` carries - # only `(table_key, field_key)` STRINGS out of this scope. The mutation below takes its own - # strict document from `rt.update` and never sees a projected value, which is C5's second - # clause: a projected snapshot is never handed to a post-write read-back. - # ⚠ HONEST RESIDUAL, STATED RATHER THAN SMOOTHED (R6's second sentence): D-179's exit condition - # asks for ZERO full-document reads and this is ONE — `rt.update` below takes a strict read - # that belongs to the WRITE and cannot be removed from here. What is gone is the expensive one. - # ⚠ AND THE FALLBACK IS DELIBERATE: `all_defs` degrades to the whole read on any failure, so a - # store that cannot project still deletes correctly, only slower. - # - # ⭐⭐ WAVE 35 · T34 — THE RESIDUAL IS NOW MEASURED RATHER THAN ASSERTED, and the count is - # better than the paragraph above claims. Whole `user_tables` reads per delete, against a - # runtime that has production's `get_projection`: - # TAGGED (a discovery flow that spawned columns) → 1 whole + 1 projected - # UNTAGGED (a `plain` flow) → 0 whole + 1 projected - # The untagged case pays NONE because `if not plan: return` fires before the write. The - # tagged case's ONE is `rt.update`'s own strict read, and it is a FLOOR, not a choice: - # `core/store.py::get_projection` states that a write must always read whole, because a - # read-modify-write handed a rows-less document would upload the tenant with its rows deleted. - # ⛔ Reaching ZERO needs D-179's `{flowId: [(table_key, field_key)]}` index, written where the - # presets are spawned — a feature, not an edit to this function. Nothing here can do better. - # ⚠ THOSE NUMBERS WERE UNOBSERVABLE UNTIL W35-T34: `verify_automation`'s delete-speed double - # had no `get_projection`, so `all_defs` fell back and the gate measured 2 and 1 — the - # PRE-WAVE-33 path — for two waves. The double is producer-faithful now and asserts the counts. - try: - import core.user_tables as _ut_defs # noqa: PLC0415 - _scan = dict(_ut_defs.all_defs(rt) or {}) - except Exception: # noqa: BLE001 - _scan = ut_all(rt) - plan = [(tk, str(f.get("key") or "")) - for tk, t in _scan.items() - for f in ((t or {}).get("fields") or []) - if str((f.get("automation") or {}).get("flowId") or "") == aid] - if not plan: - return - - def _disable(cur): - cur = cur if isinstance(cur, dict) else {} - note = f"its automation was deleted {_stamp()}" - for tk, fk in plan: - for f in ((cur.get(tk) or {}).get("fields") or []): - a = f.get("automation") - if f.get("key") == fk and isinstance(a, dict): - a["disabled"] = True - a["statusNote"] = note - return cur - # The expensive half of the owner's complaint: a whole-document rewrite, committed while the - # person waits. Async for the reason given above — the disable is visible to the next read - # immediately; only the upload is deferred, and it coalesces with any other pending write to - # the same key instead of racing it. - rt.update(UT_STORE_KEY, _disable, flush="async") - - -# --------------------------------------------------------------------------------------------- -# RUN STATE — process memory only (see the module header for why this may never be persisted) -# --------------------------------------------------------------------------------------------- - -_RUN_LOCK = threading.RLock() -_RUNNING = {} # (tenant, id) -> {'startedAt', 'step'} - - -def running(tenant, auto_id): - with _RUN_LOCK: - return dict(_RUNNING.get((tenant, str(auto_id))) or {}) or None - - -def _claim(tenant, auto_id): - with _RUN_LOCK: - if (tenant, str(auto_id)) in _RUNNING: - return False - _RUNNING[(tenant, str(auto_id))] = {"startedAt": _iso(), "step": "starting"} - return True - - -def _step(tenant, auto_id, text): - with _RUN_LOCK: - cur = _RUNNING.get((tenant, str(auto_id))) - if cur is not None: - cur["step"] = text - - -def _no_step(_text): - """The default `step` sink for a runner nobody gave a live slot to (a gate driving a runner - directly). A runner must never REQUIRE the slot — the slot is process state, and the runner - is the part a test is allowed to call on its own.""" - - -def _release(tenant, auto_id): - with _RUN_LOCK: - _RUNNING.pop((tenant, str(auto_id)), None) - - -def _commit_run(rt, auto_id, state, summary, counts, ok, affected=None, steps=None, notes=None): - """THE ONE store write a run performs on the `automations` bucket. - - `steps` is the per-NODE outcome map the canvas paints its dots from. It is recorded here - because it is a MEASUREMENT the runner took as it walked — see `graph`, which refuses to - invent a node status when this map is absent (every run stored before W19-C has no `steps`, - and painting those nodes green from the run's overall state would be a fabrication). - - ⭐⭐ `notes` CLOSES DEBT D-103 (2026-08-09). A run's `counts` are integers and the - comprehension below drops everything else — so the per-record sentence a vendor gave us had - literally nowhere to live, and *"1 profile read(s) were blocked"* was the whole of what the - product could say. MEASURED on nurilab: the same record blocked on three separate runs and - the reason was unrecoverable from the store afterwards, so the diagnosis had to be rebuilt by - calling the vendors by hand. The note is the most valuable thing a run produces — a dead - handle, a vendor hiccup and an exhausted key are three different actions — and it was the one - thing thrown away. - ⚠ ON THE RUN, NOT ON THE RECORD. D-103's own prescription: no tenant table gains a column it - did not ask for, and W25/R4 already retired writing status STRINGS into people's grids. - """ - entry = {"ts": _iso(), "ok": bool(ok), "summary": _s(summary, 400), - "counts": {k: int(v) for k, v in (counts or {}).items() - if isinstance(v, (int, float))}, - "steps": {str(k): str(v) for k, v in (steps or {}).items()}, - # Bounded on both axes: a 100-profile run must not put 100 sentences into a store - # entry that is kept 20 deep per automation. - "notes": [_s(n, 300) for n in (notes or []) if str(n or "").strip()][:25], - "affected": list(affected or [])[:200]} - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - d = cur.get(str(auto_id)) - if d is None: - return cur - d["status"] = {"state": state, "lastRunAt": entry["ts"], - "lastSummary": entry["summary"]} - d["runs"] = ([entry] + list(d.get("runs") or []))[:MAX_RUNS] - # Wave 22 (airtable-brief rec 6): K consecutive FAILURES auto-pause the automation with - # the reason on it — a dead credential must not burn quota (or a paid vendor's records) - # 96 times a day while a green toggle sits over a buried error log. Errors only: - # `partial` is the honest-progress state and pausing on it would punish honesty. - runs = d.get("runs") or [] - if state == "error" and len(runs) >= CONSECUTIVE_FAILURE_PAUSE and all( - not r.get("ok") for r in runs[:CONSECUTIVE_FAILURE_PAUSE]): - sch = d.get("schedule") - if isinstance(sch, dict) and sch.get("enabled"): - sch["enabled"] = False - trg = d.get("trigger") - if isinstance(trg, dict): - trg["paused"] = True - d["statusNote"] = (f"auto-paused after {CONSECUTIVE_FAILURE_PAUSE} consecutive " - f"failures. Fix the cause, then re-enable: " - f"{entry['summary'][:120]}") - return cur - - _store_update(rt, _up, flush="sync") - _notify_outcome(rt, auto_id, state, entry) - return entry - - -def _notify_outcome(rt, auto_id, state, entry): - """⭐⭐ WAVE 27 ITEM 31 / CONTRACT C5 — the bell rings on EVERY outcome, good and bad (I7). - - Owner: notify on both success and error. `_commit_run` is the ONE place a run's outcome is - written, so it is the only place this can go without a second definition of "what happened". - - ⛔ AMENDMENT A1 MOVED TWO REQUIREMENTS HERE AND THE FIRST IS A CROSS-TENANT DEFECT IF MISSED. - 1. **`st=` IS PASSED EXPLICITLY.** `core.alerts.notify`'s `st=None` default resolves to the - MODULE-level store, which is tenant #0's (Royal Imports). The automation tick runs on a - background thread with no session, so an omitted `st` files nurilab's automation failures - in Royal Imports' inbox — D-16's class, silently, forever. - 2. **THE OWNER IS THE AUTOMATION'S `createdBy`, NEVER THE RUNNER'S IDENTITY.** The scheduler - has no session at all, so the alternative is not "the wrong person" but "nobody", and - `notify` returns None on a blank owner — the notification would simply never exist, on - exactly the runs nobody was watching. - ⚠ AND NO SERVER ADMISSION WAS NEEDED. A1 measured that `routes_alerts._TOPICS` gates only - view-alert CREATION; `notify()` writes straight into the notifications bucket and - `GET /notifications` filters by nothing, so an `automation` topic already reaches the bell. - ⚠ FAILURE HERE IS SWALLOWED. A notification that cannot be filed must not turn a run that - SUCCEEDED into one that raised — the run entry is already committed one line above, and the - bell is a courtesy on top of it. - """ - try: - defn = (rt.get(STORE_KEY) or {}).get(str(auto_id)) or {} - owner = str(defn.get("createdBy") or "").strip() - if not owner: - return - from core import alerts as _alerts - _alerts.notify( - owner=owner, - # ⭐ WAVE 34 · R12 — the fallback name a notification wears when the automation has - # none. "Agent", singular: this names ONE of them, not the module. - label=_s(defn.get("name") or "Agent", 80), - topic="automation", - key=str(auto_id), - detail=f"{state}: {entry.get('summary') or ''}"[:300], - st=rt) - except Exception: # noqa: BLE001 - pass - - -# --------------------------------------------------------------------------------------------- -# INSTAGRAM (R7, extended by R1) — PUBLIC DATA ONLY, NEVER AN INSTAGRAM LOGIN -# --------------------------------------------------------------------------------------------- -# ⚠ AMENDED W19-C, vendor swapped W20. This section read "ANONYMOUS PUBLIC ENDPOINTS ONLY" and -# that is no longer the whole truth: R1 added a paid VENDOR rung (Bright Data, further down) -# which does authenticate — to the vendor. The rail that matters is unchanged and is sharper: -# -# **NOTHING HERE EVER AUTHENTICATES TO INSTAGRAM.** No login, no password, no session cookie, -# no account to get banned, public data only. The vendor key is a key to a SUPPLIER, and the -# supplier takes the scraping risk; it is never an Instagram credential and this module must -# never be given one. -# -# The rungs below this line are the ANONYMOUS ladder ($0, no key of any kind). -# -# ⚠ HONEST STATUSES ARE THE FEATURE. Instagram rate-limits and outright blocks datacenter egress -# (the HF Space and any AWS Lambda are both in that class), and its anonymous surface has been -# progressively closed for years. So this returns a STATE — ok / partial / blocked / error — and -# the cell says which. A pull that quietly wrote zero posts and reported success would be worse -# than no automation at all: the table would look maintained and be empty. -# -# THE LADDER, tried in order with pacing between rungs. Each rung records how it did in `via`, -# so a run history answers "what still works anonymously?" from data rather than from memory. - -PACE_SECONDS = float(os.environ.get("AIOS_IG_PACE_SECONDS") or 2.5) - -#: ⭐ MAP THE SCHEMA, NOT JUST WHAT WAS POPULATED ON THE PROBE — and the reason is this page's -#: own headline. **HISTORY IS UNBUYABLE.** No vendor sells "followers on 1 January"; every number -#: is a NOW value, so the series starts the day capture starts and a field we do not capture -#: TODAY is permanently lost for today. Dropping a column because it was null on two probe rows -#: has exactly the same cost as delaying capture — for that column — and is justified by exactly -#: the evidence (n=2) that was judged too thin to close D-25. Adding a column that turns out to -#: be usually-empty costs a blank cell; omitting one costs the months before somebody notices. -#: Every returned provider field is retained in Source data. Promoted columns make commonly used -#: values filterable; the raw source document prevents a new or uncommon field from being lost. -#: ⚠ A blank in any of these means NOT READ — never "they have none". The anonymous rungs expose -#: almost none of them, which is what the `source` column is for. -SNAPSHOT_FIELDS = [ - field_def("snapshot_key", "Snapshot"), field_def("influencer_key", "Influencer"), - field_def("pulled_at", "Pulled at", "date"), field_def("followers", "Followers", "int"), - field_def("following", "Following", "int"), - field_def("posts_count", "Post count", "int"), - field_def("full_name", "Name"), field_def("bio", "Bio"), - field_def("verified", "Verified", "checkbox"), field_def("source", "Read via"), - field_def("approx", "Counts are approximate", "checkbox"), - # The link in bio — the paid rung's field, and commercially the most useful single string an - # influencer row can carry (it is the shop/affiliate destination). - field_def("external_url", "Link in bio", "url"), - # --- the rest of the vendor's Profiles schema. - field_def("ig_id", "Instagram id"), - field_def("profile_url", "Profile", "url"), - field_def("avg_engagement", "Avg engagement", "pct"), - field_def("category", "Category"), - field_def("business_category", "Business category"), - field_def("is_business", "Business account", "checkbox"), - field_def("is_professional", "Professional account", "checkbox"), - field_def("is_private", "Private", "checkbox"), - field_def("highlights_count", "Story highlight count", "int"), - field_def("bio_hashtags", "Bio hashtags"), - field_def("pronouns", "Pronouns"), - # ⭐ 2026-08-07 — the rest of the vendor's Profiles schema (see `_bd_profile`). The snapshot - # row maps the WHOLE schema by design, so these belong here the moment the map reads them; - # leaving them out would be the "captured but unrecorded" half of the same loss the note at - # the top of this list is about. - field_def("profile_name", "Profile name"), - field_def("is_joined_recently", "Joined recently", "checkbox"), - field_def("has_channel", "Has channel", "checkbox"), - field_def("partner_id", "Partner id"), - field_def("external_url_title", "Link title"), - field_def("fbid", "Facebook id"), - field_def("related_accounts", "Related accounts"), - field_def("country_code", "Country"), - field_def("source_payload", "Source data", "json"), -] -POST_FIELDS = [ - field_def("shortcode", "Shortcode"), field_def("influencer_key", "Influencer"), - field_def("posted_at", "Posted at", "date"), - field_def("type", "Type", "select", options=["image", "video", "carousel"]), - field_def("caption", "Caption"), field_def("url", "URL", "url"), - # A tagged place is useful content context and can be filtered as ordinary text. It is never - # presented as the creator's location: a creator can tag a holiday, venue or brand location. - field_def("tagged_location", "Tagged location"), - # The per-field map below gives operational fields first; this locked JSON document retains - # every other value Bright Data supplied, so a vendor schema addition is preserved - # instead of being silently discarded while the canonical field model catches up. - field_def("source_payload", "Source data", "json"), - # Sponsored-post detection — §2c called it one of the fields worth having that the anonymous - # ladder cannot reach, and it came back MEASURED-populated (`True`, with the brand attached). - field_def("paid_partnership", "Paid partnership", "checkbox"), - field_def("partner", "Partner brand"), - field_def("hashtags", "Hashtags"), - field_def("alt_text", "Alt text"), - # ⭐⭐ 2026-08-07 — THE LATEST-VALUE ENGAGEMENT COLUMNS, AND THEY EXIST TO KEEP A ROLLUP AT - # ONE HOP. - # - # "Average views over the last N posts" is naturally TWO hops: profile → posts → each post's - # most recent snapshot → `views`. Airtable's rollup is one hop, and growing a second one is a - # much bigger build with a much worse failure mode (a rollup over a rollup, invalidated - # transitively). So the post row carries its own LATEST value and the rollup reads it - # directly. - # - # ⛔ R3's "ONE STORE FOR ONE SERIES" IS UNTOUCHED, and this is exactly the pattern the profile - # row already uses one level up: LATEST on the row (+ `enriched_at` to date it), the SERIES in - # `ut_ig_post_snapshots`. These three cells are rewritten on every pull from the snapshot that - # was just appended — they are a projection of that store, never a second copy of it, and - # deleting them would cost a convenience rather than a measurement. - # ⚠ BLANK, NEVER ZERO — and `_ig_zero_is_blank` is what finally ENFORCES it. `postMetrics` is - # off by default (it buys one vendor record per post), so on most pulls this stays empty, and - # empty means "not read", which is what makes an honest average possible at all. A 0 here - # would claim a post nobody watched. ⚠ The rule is scoped to this paid rung: a zero LIKE or - # COMMENT count is a real measurement and must survive. - # - # ⛔⛔ THERE IS NO `views` COLUMN HERE, AND THAT IS A RULING, NOT AN OVERSIGHT (2026-08-08, - # instagram-capture.md §4e/§4f — owner-approved after the evidence was bought). - # Bright Data's `views` is delivered at ACCOUNT grain: one value across up to 12 distinct - # reels, on 18 of 19 creators, through BOTH collection modes, while `likes` varies richly in - # the very same rows. Mapping it here is what put one number on many unrelated posts. We - # cannot say what it measures at ANY grain (`sriyynntt` returns 0 at 11,102 followers; - # `reviewby_ayyaa` returns null at 20k–328k likes), so it is not promoted anywhere — it stays - # in Source data, unlabelled, making no claim. Nothing is lost: the payloads are retained - # whole, so if the vendor ever populates it per-reel the history is re-derivable. - # ⚠ ENUMERATED, so nobody re-opens this hoping for a differently-named field: across all 30 - # fields of a Reels row, `views` and `video_play_count` are the ONLY view-shaped keys, and the - # Posts dataset carries none at all. `plays` below IS the per-reel equivalent, correctly named - # and correctly mapped. It is blank because Meta retired the Plays metric on 2025-04-10 - # (folded into a single "Views"), not because we are reading the wrong key. - # ⭐⭐ VIEWS IS BACK (2026-08-08), AND FROM A DIFFERENT SOURCE THAN THE ONE THAT WAS RETIRED. - # The retired column was fed by Bright Data's account-grain `views`. This one is fed by the - # `ig_post_views` CAPABILITY (providers.py), which resolves to Apify's `videoPlayCount` — - # measured against a browser-read ground truth to the digit. Same column name, same type, same - # rollup; the engine underneath is swappable and the preset database never moved. That is the - # owner's schema ruling working exactly as intended. - # ⚠ STILL BLANK, NEVER ZERO, and still only on the paid rung. - field_def("views", "Views", "int"), - field_def("plays", "Plays", "int"), - field_def("likes", "Likes", "int"), - field_def("comments", "Comments", "int"), - # ⭐⭐ 2026-08-09 (owner: *"whatever APIfy has more than BD pls use it and fix the post data - # further"*). Two fields the primary source does not return AT ALL, already paid for inside - # the same engagement response — so capturing them costs nothing extra. - # ⛔ A COLUMN MUST EXIST BEFORE A CELL CAN BE WRITTEN. `user_tables` filters an unknown key on - # every write door, so a normaliser that emits `video_duration` without this line writes - # nothing and reports success — the wave-28 defect that swallowed nine preset cells. - field_def("video_duration", "Video length (s)", "int"), - field_def("comments_disabled", "Comments off", "checkbox"), - field_def("measured_at", "Engagement read at", "date"), - # ⭐ 2026-08-07 — the second half of "spawn relevant Post/Comment database that is LINKED": - # a post reaches its own comment rows the same way a profile reaches its posts. Derived from - # `shortcode`, so it is correct the moment a comment row exists and needs no maintenance. - # ⚠ It resolves to nothing until comment capture is switched on, which is the honest state - # for a relation whose far side is empty — the same standing `ut_ig_post_snapshots` has when - # `postMetrics` is off. - field_def("comments_link", "Comment rows", "link", - description="Comment records linked to this post.", - link={"table": IG_COMMENTS_TABLE, "on": "shortcode", "from": "shortcode"}), - field_def("post_snapshots_link", "Measurement rows", "link", - description="Engagement measurements linked to this post.", - link={"table": IG_POST_SNAPSHOTS_TABLE, "on": "shortcode", "from": "shortcode"}), - field_def("measurements_captured", "Measurements captured", "rollup", - rollup={"link": "post_snapshots_link", "fn": "countall"}), -] - -#: ⭐⭐ 2026-08-07 (owner instruction) — THE COMMENT DATABASE. -#: -#: Owner: *"an enrichment automation should spawn relevant Post/Comment database that is linked to -#: the profile automatically."* So the schema and the LINK ship; what does NOT ship on by default -#: is the capture. -#: -#: ⛔ THIS REVERSES A STANDING RULING (D-22 / R11) AND THE REVERSAL IS DELIBERATE AND BOUNDED. -#: Comment capture was refused on two grounds, and BOTH ARE STILL TRUE: -#: 1. COST — comments are ~98% of a full-history bill (~$56,000 at 37.5M comments, §2a of -#: `instagram-capture.md`). They are the single most expensive thing this product can buy. -#: 2. PRIVACY — Bright Data flags `comment_user` and `post_user` as PII, and the append law has -#: no erasure path for third parties who never appeared in the tracked set (D-24). A comment -#: thread ingests identifiable people who never entered anybody's influencer list. -#: ⛔ SO WHAT SHIPS HERE IS THE SCHEMA AND THE LINK. **NOTHING CAPTURES COMMENTS TODAY** — no code -#: calls Bright Data's Comments dataset (`gd_ltppn085pokosxh13`), there is no `config.comments` -#: switch, and this table stays EMPTY until somebody builds one. Stated in the present tense on -#: purpose: an earlier draft of this note described the opt-in switch as if it existed, which is -#: exactly the doc that "reads as authority and silently ages" that §0 of `instagram-capture.md` -#: is written against. -#: ⚠ WHEN IT IS BUILT it should be an OPT-IN defaulted OFF, the same posture as -#: `config.postMetrics` and the Bright Data money switch W26/R15 kept — the relation costs -#: nothing; the spend and the third-party ingest are a click somebody makes knowingly. -#: Embedded comment payloads delivered with a Profile/Post/Reel response are retained because -#: they are part of a record already paid for. The separate full Comments dataset remains OFF -#: unless `commentMetrics` is explicitly enabled on the enrichment action. -COMMENT_FIELDS = [ - field_def("comment_key", "Comment"), field_def("shortcode", "Shortcode"), - field_def("influencer_key", "Influencer"), - # ⭐⭐ OWNER RULING 2026-08-12 — the same one that put `text` on the TikTok comment schema, and - # it lands on BOTH networks in the same change on purpose: the two comment tables are read side - # by side, and one carrying the content while the other does not is the divergence this repo - # keeps paying for. ⛔ TEXT ONLY — `comment_user` / `comment_user_url` are vendor-FLAGGED PII - # and stay in `source_payload`, uncolumned. - field_def("text", "Comment"), - field_def("commented_at", "Commented at", "date"), - field_def("likes", "Likes", "int"), - field_def("replies", "Replies", "int"), - field_def("source_payload", "Source data", "json"), - field_def("post_link", "Post", "link", - description="Post record linked to this comment.", - link={"table": IG_POSTS_TABLE, "on": "shortcode", "from": "shortcode", - "single": True}), - # Author/text and every provider-specific value stay intact in Source data. The promoted - # columns intentionally keep the grid concise while Likes and Replies make comment engagement - # directly filterable and rollup-ready. -] -POST_SNAPSHOT_FIELDS = [ - field_def("post_snapshot_key", "Snapshot"), field_def("shortcode", "Shortcode"), - field_def("influencer_key", "Influencer"), - field_def("pulled_at", "Pulled at", "date"), field_def("likes", "Likes", "int"), - field_def("comments", "Comments", "int"), - # ⭐⭐ 2026-08-10 — TWO DENORMALISED POST FACTS, and they are worth having on their own merits - # before any rollup argument is made. - # - # `pulled_at` says when we LOOKED; `posted_at` says when the creator POSTED. A fact table with - # only the first can describe a measurement but not its AGE, so the single most useful thing - # this series can compute — views at N days old, i.e. velocity — is not expressible over it at - # all. `type` is the same shape one column over: "reels only" is the default lens on this data - # and without it every question has to go back through `ut_ig_posts` to ask what kind of post - # this was. - # - # ⚠ THE WRITER HAS BOTH IN HAND (`capture_rows` builds the identity row and the measurement row - # from the same vendor record), so this costs one dict key each and no extra call. - # - # ⛔ `options` IS NOT OPTIONAL ON A `select`. A select declaring none is the wave-26 item-24 - # shape: the filter panel receives an empty list as an ANSWER, `if ([])` is truthy, and the - # control renders dead with nothing to say. Same three values `POST_FIELDS` declares — copied - # from it deliberately rather than shared, because these are two different tables' columns that - # happen to agree today, and `verify_automation` asserts the agreement. - # ⛔⛔ AND THEY DO NOT MAKE "THE LAST 10 REELS" EXPRESSIBLE OVER THIS TABLE. That takes TWO - # orderings (newest measurement per post, then newest posts), the rollup bag has one `sortBy`, - # and `ut_ig_posts` already performs the first of them. See the verdict in - # `.claude/wiki/research/entity-vs-series.md` — these columns are for AGE and KIND, not for - # moving the window here. - field_def("posted_at", "Posted at", "date"), - field_def("type", "Type", "select", options=["image", "video", "carousel"]), - # Plays move independently of likes on video, so it is its own series rather than a thing to - # derive. Paid rung only; blank means not read. - # The series regains Views alongside the latest-value projection on the Post row — one store - # for one series (R3) is untouched; this IS that store. - field_def("views", "Views", "int"), - field_def("plays", "Plays", "int"), - field_def("source_payload", "Source data", "json"), - field_def("post_link", "Post", "link", - description="Post record linked to this measurement.", - link={"table": IG_POSTS_TABLE, "on": "shortcode", "from": "shortcode", - "single": True}), -] - -#: ⭐⭐ WAVE 32 · T41 — THE INSTAGRAM TWIN OF `TT_TABLE_FIELDS`, WHICH DID NOT EXIST. -#: -#: `TT_TABLE_FIELDS`' own note said so and named the consequence: *"The Instagram side has no -#: equivalent map, which is exactly why its table names are scattered across the module."* It is a -#: map now, for a concrete reason rather than symmetry: A's boot-time delivery sweep (`W32-T07`) has -#: to call `ut_ensure` for every locked child of BOTH platforms, and a sweep that can ask TikTok for -#: its table set and must hand-type Instagram's is one platform away from the drift this pair of -#: maps exists to stop. -#: -#: ⛔ THESE FOUR LISTS ARE THE *CANONICAL* DECLARATION — the per-tenant BACKLINK fields -#: `ensure_ig_graph` appends are NOT here, and must not be. A backlink is derived per profile -#: database (`_profile_backlink_field`), so folding it into a module constant would make one -#: tenant's relation a global fact. `ensure_ig_graph` reads this map and adds its own backlinks on -#: top, which is why that function still owns the graph write and this only owns the schema. -IG_TABLE_FIELDS = { - IG_SNAPSHOTS_TABLE: SNAPSHOT_FIELDS, - IG_POSTS_TABLE: POST_FIELDS, - IG_POST_SNAPSHOTS_TABLE: POST_SNAPSHOT_FIELDS, - IG_COMMENTS_TABLE: COMMENT_FIELDS, -} -#: The labels those four wear in the database list. ⚠ LIFTED VERBATIM from `ensure_ig_graph`'s own -#: `graph` literal, which is now derived from this map — a renamed label here renames the table -#: everywhere rather than leaving two spellings of one database. -IG_TABLE_LABELS = { - IG_SNAPSHOTS_TABLE: "IG snapshots", - IG_POSTS_TABLE: "IG posts", - IG_POST_SNAPSHOTS_TABLE: "IG post snapshots", - IG_COMMENTS_TABLE: "IG comments", -} - - -# --------------------------------------------------------------------------------------------- -# ⭐⭐ TIKTOK — THE FIVE `ut_tt_*` SCHEMAS (wave 29 · item 7 · D-9 · rulings R1 + R2) -# --------------------------------------------------------------------------------------------- -# ⛔ EVERY VENDOR FIELD NAME BELOW IS FROM ONE PROBED SOURCE — `waves/wave29/proto/tiktok-schema.md`, -# 40 profile / 43 post / 17 comment fields read live from `GET /datasets/{id}/metadata` for $0.00, -# each carrying the vendor's own type, description and `pii` flag. NOTHING HERE IS GUESSED, and the -# names live in `connectors_tt.py` (the map), not here (the schema). This file declares what a -# COLUMN is; the connector declares what the wire calls it. -# -# ⭐ THE SAME LAW AS THE INSTAGRAM LISTS ABOVE: map the schema, not just what was populated on the -# probe. HISTORY IS UNBUYABLE — no vendor sells "followers on 1 January" — so a field we do not -# capture today is permanently lost for today, and an occasionally-empty column costs a blank cell -# while a missing one costs the months before somebody notices. -# -# ⚠ TWO DELIBERATE DIVERGENCES FROM THE INSTAGRAM SCHEMA, both licensed by R2 ("the two schemas may -# diverge where the vendors do") and both recorded so neither reads as an omission: -# -# 1. NO `plays` COLUMN. TikTok returns ONE number, `play_count`, and it is the count TikTok -# itself displays under a video — i.e. our `views`. Instagram has two columns because Meta -# once had two metrics (`plays` retired 2025-04-10). Writing one vendor number into two of our -# columns would manufacture a second measurement that a rollup could average or double-count, -# and blank-vs-zero discipline says an unmeasured column must be blank, not a copy. -# ⇒ `play_count` -> `views`, and `plays` does not exist on this family. -# 2. `tt_id`, NOT `ig_id`. The probe doc flags this: our profile key is literally NAMED `ig_id`. -# A TikTok row carrying a column labelled "Instagram id" is a header that lies, and this is a -# NEW table family with no stored rows to migrate — so the honest name costs nothing. -# -# ⚠ THE COLUMNS THAT STAY BLANK ARE NAMED RATHER THAN DROPPED SILENTLY. TikTok has no equivalent of -# `category`, `business_category`, `is_professional`, `highlights_count`, `bio_hashtags`, -# `pronouns`, `profile_name`, `is_joined_recently`, `has_channel`, `partner_id`, -# `external_url_title`, `fbid` or `related_accounts` (profile), nor of `alt_text`, -# `comments_disabled`, `paid_partnership` or `partner` (post) — `commerce_info` is a LOCATION, not -# a paid-partnership flag, and TikTok's `comment_setting` is profile-level rather than per-post. -# Those columns are ABSENT here rather than declared and permanently blank — a column nothing can -# ever write is a promise the grid keeps making that the vendor cannot keep. - -#: ⭐ WAVE 29 (contract C2) — the value the `profile` FLAG carries on a TikTok handle column: the -#: twin of `PROFILE_SOURCE_IG`, which is declared further down beside the `PLATFORM_*` vocabulary -#: with the note on why those two families of string are NOT the same thing. It sits here rather -#: than there because the lists below are evaluated at IMPORT and would raise a NameError otherwise. -#: ⛔ THIS CONSTANT IS ONLY HALF THE CONTRACT, AND THE OTHER HALF HAS LANDED (verified 2026-08-12, -#: W30-T08): `platform/core/user_tables.py:PROFILE_SOURCES` now reads `('instagram', 'tiktok')`. -#: Before it did, `_clean_profile` REFUSED this flag and the handle column of a `ut_tt_profile` -#: spawn was dropped — fail-closed and loud by design, because the alternative is a profile -#: database whose profile column silently is not one. Left written down rather than deleted: the -#: refusal is what a TikTok binding looks like on any deployment where that line is missing. -PROFILE_SOURCE_TT = "tiktok" - -#: The profile row: LATEST values plus `enriched_at`. The SERIES lives in `ut_tt_snapshots` — -#: R3's "one store for one series" carries over unchanged, because the reason for it does (no -#: vendor sells history, so the append table is the only history there will ever be). -TT_PROFILE_FIELDS = [ - # `platform` FIRST, exactly as the Instagram preset set has it — and on all five tables here, - # not just this one. The IG family carries it on the profile table alone, which is why a TikTok - # POST could never have lived beside an Instagram post; declaring it everywhere is what makes a - # future cross-platform union view a UNION rather than a guess. - tt_field_def("platform", "Platform"), - tt_field_def("handle", "Handle", pinned=True, - profile={"source": PROFILE_SOURCE_TT}), - tt_field_def("full_name", "Name"), - tt_field_def("tt_id", "TikTok id"), - tt_field_def("profile_url", "Profile", "url"), - tt_field_def("bio", "Bio"), - tt_field_def("external_url", "Link in bio", "url"), - tt_field_def("verified", "Verified", "checkbox"), - tt_field_def("is_private", "Private", "checkbox"), - tt_field_def("is_business", "Business account", "checkbox"), - tt_field_def("followers", "Followers", "int"), - tt_field_def("following", "Following", "int"), - tt_field_def("posts_count", "Video count", "int"), - # TikTok gives THREE engagement rates where Instagram gives one. All three are stored ×100 for - # the same reason `avg_engagement` is on the IG side (C1-a): the vendor sends a 0–1 fraction and - # our `pct` renderer appends the sign to the stored number, so the raw fraction would print a - # 6.6% creator as `0.0%`. - tt_field_def("avg_engagement", "Avg engagement", "pct"), - tt_field_def("like_engagement", "Like engagement", "pct"), - tt_field_def("comment_engagement", "Comment engagement", "pct"), - # Total likes RECEIVED across the account's videos — a TikTok-only number with no Instagram - # equivalent, and one of the few profile-level engagement facts a vendor gives away. - tt_field_def("likes_received", "Likes received", "int"), - tt_field_def("country_code", "Country"), - tt_field_def("region", "Region"), - tt_field_def("predicted_lang", "Language"), - # ⚠ ACCOUNT AGE, NOT A MEASUREMENT STAMP. The vendor's `create_time` on a profile is when the - # ACCOUNT was created; it is emphatically not "when `followers` was true". TikTok carries no - # measurement timestamp at all, exactly like Instagram — which is why the append law below - # (`@`) is the only thing that can date a number. - tt_field_def("account_created_at", "Account created", "date"), - tt_field_def("first_found", "First found", "date"), - tt_field_def("last_found", "Last found", "date"), - tt_field_def("found_count", "Times found", "int"), - tt_field_def("created_by", "Found by"), - tt_field_def("enriched_at", "Enriched at", "date"), - tt_field_def("source", "Read via"), - tt_field_def("source_payload", "Source data", "json"), - tt_field_def("profile_snapshots_link", "Measurement rows", "link", - description="Profile measurements linked to this account.", - link={"table": TT_SNAPSHOTS_TABLE, "on": "influencer_key", "from": "handle"}), - tt_field_def("posts_link", "Post rows", "link", - description="Post records linked to this account.", - link={"table": TT_POSTS_TABLE, "on": "influencer_key", "from": "handle"}), -] - -#: The profile SERIES. One row per profile per pull, keyed `@` — the append law -#: from `instagram-capture.md` §3, unchanged, because its cause is unchanged: the vendor stamps -#: nothing, so the only honest date a number can carry is the moment WE read it. -TT_SNAPSHOT_FIELDS = [ - tt_field_def("platform", "Platform"), - tt_field_def("snapshot_key", "Snapshot"), - tt_field_def("influencer_key", "Influencer"), - tt_field_def("pulled_at", "Pulled at", "date"), - tt_field_def("followers", "Followers", "int"), - tt_field_def("following", "Following", "int"), - tt_field_def("posts_count", "Video count", "int"), - tt_field_def("likes_received", "Likes received", "int"), - tt_field_def("full_name", "Name"), - tt_field_def("bio", "Bio"), - tt_field_def("verified", "Verified", "checkbox"), - tt_field_def("is_private", "Private", "checkbox"), - tt_field_def("is_business", "Business account", "checkbox"), - tt_field_def("avg_engagement", "Avg engagement", "pct"), - tt_field_def("like_engagement", "Like engagement", "pct"), - tt_field_def("comment_engagement", "Comment engagement", "pct"), - tt_field_def("external_url", "Link in bio", "url"), - tt_field_def("tt_id", "TikTok id"), - tt_field_def("profile_url", "Profile", "url"), - tt_field_def("country_code", "Country"), - tt_field_def("region", "Region"), - tt_field_def("predicted_lang", "Language"), - tt_field_def("account_created_at", "Account created", "date"), - tt_field_def("source", "Read via"), - tt_field_def("approx", "Counts are approximate", "checkbox", - description="Checked when the counts on this row are rounded, not exact."), - tt_field_def("source_payload", "Source data", "json"), -] - -#: One row per TikTok post. Identity is `shortcode`, and on TikTok that is a 19-digit numeric id — -#: ✅ the SAME shape as the Comments dataset's `post_id`, so the comments→posts link joins on -#: equality with NO normaliser (measured in the probe doc; Instagram needed one). -TT_POST_FIELDS = [ - tt_field_def("platform", "Platform"), - tt_field_def("shortcode", "Post id"), - tt_field_def("influencer_key", "Influencer"), - tt_field_def("posted_at", "Posted at", "date"), - # ⛔ THE VENDOR'S VOCABULARY IS `"video"` / `"content"` AND OURS HAS NO `"content"`. Declaring - # the vendor's word would put an untranslated API token in front of a user; declaring an option - # the mapper can emit but the select does not list would fail `_clean_field`. So the OPTIONS - # stay this product's words and `connectors_tt.normalize_post` does the translation — the same - # posture every other vendor value here takes. `carousel` is listed because a TikTok photo post - # carrying more than one image IS one, and the mapper decides from `carousel_images` rather - # than from the type token, which cannot express it. - tt_field_def("type", "Type", "select", options=["image", "video", "carousel"]), - tt_field_def("caption", "Caption"), - tt_field_def("url", "URL", "url"), - tt_field_def("hashtags", "Hashtags"), - tt_field_def("tagged_location", "Commerce location"), - # ⛔ `views` AND NO `plays` — see the divergence note at the top of this section. One vendor - # number, one column. - tt_field_def("views", "Views", "int"), - tt_field_def("likes", "Likes", "int"), - tt_field_def("comments", "Comments", "int"), - tt_field_def("shares", "Shares", "int"), - tt_field_def("saves", "Saves", "int"), - tt_field_def("video_duration", "Video length (s)", "int"), - tt_field_def("measured_at", "Engagement read at", "date", - description="When the engagement numbers on this row were read."), - tt_field_def("source_payload", "Source data", "json"), - tt_field_def("comments_link", "Comment rows", "link", - description="Comment records linked to this post.", - link={"table": TT_COMMENTS_TABLE, "on": "shortcode", "from": "shortcode"}), - tt_field_def("post_snapshots_link", "Measurement rows", "link", - description="Engagement measurements linked to this post.", - link={"table": TT_POST_SNAPSHOTS_TABLE, "on": "shortcode", "from": "shortcode"}), - tt_field_def("measurements_captured", "Measurements captured", "rollup", - description="How many measurements this post has.", - rollup={"link": "post_snapshots_link", "fn": "countall"}), -] - -#: The POST series. ⚠ D-117 is live on the Instagram twin — `posted_at`/`type` are denormalised onto -#: snapshot rows there and never reconcile with the post row afterwards. That pattern is NOT -#: reproduced: this table carries the measurement and its identity, and asks `ut_tt_posts` for what -#: kind of post it was. A fact that can disagree with its own dimension table is a fact nobody can -#: trust, and the convenience it buys (one fewer hop in a rollup) is not worth a column that can be -#: wrong. -TT_POST_SNAPSHOT_FIELDS = [ - tt_field_def("platform", "Platform"), - tt_field_def("post_snapshot_key", "Snapshot"), - tt_field_def("shortcode", "Post id"), - tt_field_def("influencer_key", "Influencer"), - tt_field_def("pulled_at", "Pulled at", "date"), - tt_field_def("views", "Views", "int"), - tt_field_def("likes", "Likes", "int"), - tt_field_def("comments", "Comments", "int"), - tt_field_def("shares", "Shares", "int"), - tt_field_def("saves", "Saves", "int"), - tt_field_def("source_payload", "Source data", "json"), - tt_field_def("post_link", "Post", "link", - description="Post record linked to this measurement.", - link={"table": TT_POSTS_TABLE, "on": "shortcode", "from": "shortcode", - "single": True}), -] - -#: Comments. Same posture as Instagram's: the SCHEMA and the LINK ship, the CAPTURE is opt-in and -#: defaults OFF (`commentMetrics`), because comments are the single most expensive thing this -#: product can buy and they ingest identifiable third parties who never entered anybody's list -#: (D-24). ⚠ TikTok's `replies` is an ARRAY where ours is an INT count — a name collision, resolved -#: in the mapper by storing `num_replies` and leaving the array in Source data. -TT_COMMENT_FIELDS = [ - tt_field_def("platform", "Platform"), - tt_field_def("comment_key", "Comment"), - tt_field_def("shortcode", "Post id"), - tt_field_def("influencer_key", "Influencer"), - # ⭐⭐ OWNER RULING 2026-08-12, verbatim: *"why isn't any of the comments column actually HAS - # the comments content, fix it."* The text was bought, stored and INVISIBLE — retained whole in - # `source_payload` under the 2026-08-07 superseding ruling (`instagram-capture.md` §4b), but - # promoted to no column, so a person paying per comment record saw only Likes and Replies. - # ⛔ THE COMMENT TEXT ONLY — the commenter's IDENTITY (`commenter_user_name`, `commenter_id`, - # `commenter_url`; the vendor FLAGS the first as PII) stays in `source_payload` and gets no - # column. The ruling names the comments' CONTENT, and promoting a third party's name and - # profile URL into a filterable, exportable column is a different decision that nobody made. - tt_field_def("text", "Comment"), - tt_field_def("commented_at", "Commented at", "date"), - tt_field_def("likes", "Likes", "int"), - tt_field_def("replies", "Replies", "int"), - tt_field_def("source_payload", "Source data", "json"), - tt_field_def("post_link", "Post", "link", - description="Post record linked to this comment.", - link={"table": TT_POSTS_TABLE, "on": "shortcode", "from": "shortcode", - "single": True}), -] - -#: table key -> the field list that defines it. ⭐ DERIVED CONSUMPTION IS THE POINT: `ut_ensure`, -#: the gates and every future runner read THIS rather than naming five constants, so adding a sixth -#: `ut_tt_*` table is one entry instead of a sweep. The Instagram side has no equivalent map, which -#: is exactly why its table names are scattered across the module. -TT_TABLE_FIELDS = { - TT_PROFILE_TABLE: TT_PROFILE_FIELDS, - TT_SNAPSHOTS_TABLE: TT_SNAPSHOT_FIELDS, - TT_POSTS_TABLE: TT_POST_FIELDS, - TT_POST_SNAPSHOTS_TABLE: TT_POST_SNAPSHOT_FIELDS, - TT_COMMENTS_TABLE: TT_COMMENT_FIELDS, -} -#: The human labels a spawned `ut_tt_*` table wears in the database list. -TT_TABLE_LABELS = { - TT_PROFILE_TABLE: "TikTok profiles", - TT_SNAPSHOTS_TABLE: "TikTok profile measurements", - TT_POSTS_TABLE: "TikTok posts", - TT_POST_SNAPSHOTS_TABLE: "TikTok post measurements", - TT_COMMENTS_TABLE: "TikTok comments", -} -#: ⭐⭐ WAVE 31 · OWNER RULING R9 — the `ut_tt_*` children a person may not type records into. -#: Owner, verbatim: *"Tiktok database for post and comments and their snapshots should also be a -#: locked database with the lock icon, exactly like how instagram"*. -#: -#: ⛔ DECLARED ONCE BECAUSE THE DEFECT WAS TWO CALL SITES DISAGREEING WITH A THIRD. `ensure_ig_graph` -#: passes `record_mode=AUTOMATION_RECORD_MODE` for all four Instagram children; TikTok's two spawn -#: sites — `_spawn_presets`' child loop (SAVE) and `_tt_write_tables` (RUN) — each passed -#: `lock_fields=True` and no `record_mode` at all. That is the whole bug: not a missing feature, one -#: argument missing at two places, which is exactly the shape `_tt_write_tables`' own docstring warns -#: about (*"one function, two callers, so the inline and deferred paths cannot answer differently"*). -#: A SET plus a resolver means a sixth `ut_tt_*` table joins by declaration, not by remembering. -#: -#: ⚠ FOUR, NOT THE THREE R9 ENUMERATES, and the fourth is argued rather than assumed. -#: `ut_tt_snapshots` is the profile measurement series — the exact twin of `ut_ig_snapshots`, which -#: IS locked. R9's own comparison is *"exactly like how instagram"*, and the alternative is to leave -#: a machine-append time series that a person can type into, one table away from three that they -#: cannot. That is the fix-applied-to-one-platform-and-not-its-twin shape this very wave is closing -#: elsewhere (D-177(c)). `ut_tt_profile` is deliberately ABSENT: it is the database a human adds -#: handles to, and its Instagram counterpart is not locked either. -#: -#: ⚠ LOCKED DATABASE, NOT READ-ONLY (DESIGN.md §4): `records_mutable` goes False, so records cannot -#: be added/edited/deleted — **adding a FIELD must still work**, and a UI hiding add-field here is a -#: defect, not the intent. -TT_LOCKED_TABLES = frozenset({TT_SNAPSHOTS_TABLE, TT_POSTS_TABLE, TT_POST_SNAPSHOTS_TABLE, - TT_COMMENTS_TABLE}) -#: ⭐ THE INSTAGRAM HALF, NAMED. It was only ever the four keys `ensure_ig_graph`'s loop happens to -#: iterate, which is a set that exists but cannot be ASKED — so nothing could assert that IG and -#: TikTok lock the same shape, and the QA sweep below could not be written at all. -IG_LOCKED_TABLES = frozenset({IG_SNAPSHOTS_TABLE, IG_POSTS_TABLE, IG_POST_SNAPSHOTS_TABLE, - IG_COMMENTS_TABLE}) -#: ⭐⭐ W31 QA — OWNER, VERBATIM (2026-08-13): *"just like Instagram Post database (which is -#: locked), only the IG Profile and TT Profile should be editable."* That is this set plus its -#: complement: every `ut_ig_*`/`ut_tt_*` child is locked, and `ut_ig_profile` / `ut_tt_profile` are -#: the two a person types handles into. Registered into `core.user_tables` at import so the lock is -#: a DECLARATION rather than a flag some past automation run happened to stamp — see that module's -#: `_LOCKED_RECORD_KEYS` for why the stored flag alone left production unlocked. -#: ⛔ THE REGISTRATION ITSELF IS IN `main.py`, NOT HERE, and that is this module's own rule rather -#: than a preference: it imports NOTHING from `core` at module level (see `MAX_UT_ROWS` and -#: `MACHINE_OWNERS`, both local literals "so this module stays dependency-light for the API's boot -#: path"). Adding `import core.user_tables` at line 3570 would put `core` on the import path of -#: every process that touches the engine, to run one line. The composition root wires it, and -#: `verify_automation` asserts that main.py CARRIES that call — a declaration whose registrar is -#: missing is [[flag-shipped-without-its-writer]], which is the defect class this whole fix is in. -LOCKED_CHILD_TABLES = IG_LOCKED_TABLES | TT_LOCKED_TABLES - - -def tt_record_mode(table_key): - """R9's answer for one `ut_tt_*` key — the `record_mode` its `ut_ensure` must carry. - - `""` for anything outside the locked set, which is what `ut_ensure` already treats as "say - nothing about record mode", so an unlocked table is untouched rather than explicitly opened. - """ - return AUTOMATION_RECORD_MODE if str(table_key) in TT_LOCKED_TABLES else "" - - -# --------------------------------------------------------------------------------------------- -# THE INSTAGRAM CONNECTOR — moved out (wave 27 item 23). See `connectors_ig.py`. -# --------------------------------------------------------------------------------------------- -# Bright Data, Apify, the corpus routes and the anonymous ladder used to be ~1,900 lines HERE. -# They are a CONNECTOR: they know what a vendor's rows look like. Nothing in the automation -# runtime needs that, and the runtime is what this file is for. -# -# ⭐ THE LIST BELOW IS A DEPENDENCY STATEMENT, NOT A CONVENIENCE. It is every name the automation -# runtime still reaches for after the split — twenty, down from the seventy-one that used to be -# defined here — so `git diff` on this block is how anyone sees the engine growing a new vendor -# dependency. Re-exporting them is also REQUIRED rather than tidy: `routes_automation` and -# `routes_connectors` call `engine.bd_ready()` on the module object, and this is what keeps that -# true without those files having to learn where the wire went. -# ⚠ SO THE RE-EXPORT ALSO MEANS `engine.bd_call` STILL RESOLVES, and a gate that only checked -# that would be green whether or not anything moved. `verify_automation`'s `section_split` -# therefore asserts `__module__` on the whole moved set — it derives the split from the RUNNING -# system instead of trusting this import line ([[gate-answers-the-wrong-question]]). -# -# ⚠ AND IT IS DELIBERATELY HERE, mid-file, rather than at the top. `connectors_ig` reaches back -# for the SSRF rail (`fetch`/`fetch_json`/`Refused`) and for `PACE_SECONDS`; it does so lazily, -# inside its functions, precisely so the two modules cannot cycle. This import sits BELOW the rail -# and below `PACE_SECONDS`, so even if somebody later makes one of those reach-backs a -# module-level import, the names it wants already exist and the cycle still resolves. -# ⭐⭐ WAVE 30 · T09 (D-128) — TWO IMPORT BLOCKS NOW, AND THE SPLIT BETWEEN THEM IS THE POINT. -# `connectors_bd` is the SUPPLIER's wire — it serves both platforms and knows neither. `connectors_ig` -# is INSTAGRAM's vocabulary: its dataset ids, its row mappers, its free rungs. Anything that would -# have to change to serve a third platform belongs in the second block, not the first. -# ⚠ `bd_ready` is re-exported through this module ON PURPOSE — `routes_automation` and -# `routes_connectors` both reach `engine.bd_ready()` on the module object, and those files belong to -# other sessions. Dropping it here would 500 two surfaces that never mention a vendor. -from connectors_bd import ( # noqa: E402 - BD_EXCLUDE_MAX, BD_PATH_SNAPSHOT, BD_RECORD_PRICE_SPEC, - _bd_deferral, _bd_first_url, _bd_rows, - _first, _ig_int, - bd_call, bd_filter_rows, bd_filter_start, bd_filter_status, bd_ready, - bd_snapshot_progress, depth_refusal, -) -from connectors_ig import ( # noqa: E402 - BD_DS_COMMENTS, BD_DS_POSTS, BD_DS_PROFILES, BD_DS_REELS, - _bd_comment, _bd_post_metrics, _bd_profile, - _bd_tagged_location, - DEFERRED_MARK, - apify_profile, - bd_profiles_batch, - ig_handle, public_source, pull_profile, select_post_groups, top_up_views, -) - - -# --------------------------------------------------------------------------------------------- -# DISCOVERY — sourcing handles we do NOT already know (owner ruling R7, DEBT D-23) -# --------------------------------------------------------------------------------------------- -# The engine until now only ENRICHED a profile set somebody typed in. This queries the vendor's -# **620,000,000-record pre-collected Profiles corpus** with a real query language and returns -# handles nobody here has ever seen. MEASURED end-to-end 2026-08-04: `followers 10k–200k AND -# biography includes "floral"` returned five real profiles, one of them a verified 20.8k-follower -# Georgia/Florida floral-design company — Fisch Floral's actual market. -# -# ⛔ FIVE PROPERTIES, EACH OF THEM A MEASURED FAILURE MODE RATHER THAN A PREFERENCE: -# -# 1. **A DIFFERENT ROUTE AND A DIFFERENT NAMESPACE.** `POST /datasets/filter` — ⚠ with NO `/v3/` -# segment; `/datasets/v3/filter` is a 404 that once got written down as "the corpus is -# unreachable". Its snapshots are `snap_…` and are read at `/datasets/snapshot/…`; the -# scraper's are `sd_…` at `/datasets/v3/snapshot/…`, and the two 404 each other. -# 2. **`records_limit` IS REQUIRED.** Unbounded queries die `NOT_ENOUGH_FUNDS` (code 104, -# `per_set` billing). ⚠ And bounding is NOT sufficient: **scan time tracks PREDICATE BREADTH, -# not row count** — a `records_limit:10` query on a bare `followers > 10000` was still -# `building` 50 minutes later. So the predicate is capped too, and the runner tolerates a -# snapshot that is not ready when it looks. -# 3. **IT IS SLOW, AND THE RUN HANDS OFF RATHER THAN HOLDING A THREAD.** MEASURED delivery -# latency on the narrow query: **19.6 minutes** (`created` 12:38:00 → `delivery_time` -# 12:57:42). Blocking a worker thread for twenty minutes on a free-tier container to poll is -# the wrong shape, so a run polls for a short budget and then PERSISTS the snapshot id; the -# next run collects it. A `partial` that says "still building, the next run collects it" is -# the honest report of exactly what happened. -# 4. **PROMOTION IS MANUAL, ALWAYS (R7).** The measured result set was ~40% on-target and -# included a hashtag-aggregator account that is not a person. Auto-promoting a candidate into -# a wider profile set would silently multiply the enrichment bill AND pollute the -# snapshot series with rows nobody chose. This path never decides WHICH rows to enrich. -# 5. **SERIAL.** `429 too_many_parallel_jobs` is real and the original probe's most important -# test died on it. -# -# ⚠ THE PRICE IS NOT MEASURED AND MAY NOT BE PRESENTED AS IF IT WERE. The funds gate fires BEFORE -# the API returns a number, `price: 0` means "not priced" rather than "free", and -# `/customer/balance` answers 403 for our token — so there is no balance to read and no spend to -# report. Every estimate below is derived from the PRICING PAGE and says so. - -#: The table a discovery run writes when the tenant has no Instagram profile database yet — the -#: FIRST one, never a second (`discover_default_table` elects an existing one before this is -#: reached). Upsert by handle, so re-finding a profile is free. -#: -#: ⭐ 2026-08-10 — RENAMED FROM `ut_ig_candidates` / "IG candidates" (owner: *"We only need ONE IG -#: profile so it's not confusing"*). The old pair was two kinds of wrong at once. The KEY said -#: `candidates` while every other table in the graph says what it holds (`ut_ig_posts`, -#: `ut_ig_comments`, `ut_ig_snapshots`, `ut_ig_post_snapshots`) — and "candidate" was a concept -#: W26/R6 retired when it deleted `tracked`: a row here is a PROFILE, and whether anyone wants it -#: is a stage column's business, not the table's name. The LABEL then disagreed with the key on -#: any tenant who renamed the table, which is exactly the mismatch the owner hit. -#: -#: ⚠ SAFE TO MOVE ONLY BECAUSE NOTHING HOLDS THE OLD KEY. Census 2026-08-10 across all three -#: tenant stores: royal-imports ABSENT, gtmlab has no tables at all, and nurilab's was deleted the -#: same day (0 rows, 0 views, 0 dependent automations). An automation that stored the old key -#: explicitly still resolves to it — an explicit target is never rewritten — so a legacy tenant -#: would keep working; there simply is not one. -#: ⛔ A future rename is NOT this cheap. Once rows exist under a key, moving it is a migration -#: touching rows, `link.table`, every stored `targetTable` and the `ut__*` bucket family. -DISCOVER_TABLE = "ut_ig_profile" -#: The label that key is CREATED with. `ut_ensure` sets a label only on create and never relabels, -#: so changing this can rename nothing that already exists. One constant because it was three -#: copies of the same string literal, and a default spelled three times is one edit from drifting. -DISCOVER_LABEL = "IG profile" -#: A hard ceiling on one run's ask — and it is DERIVED, not chosen. A corpus row MEASURED at -#: ~35 KB (`file_size` 175,617 for 5 rows), so 1,000 records ≈ 33 MB would have crossed -#: `BD_MAX_KB` and come back as a truncated document **after being billed**. 500 leaves real -#: headroom. ⚠ If the row size grows, this number is the thing to re-derive. -BD_MAX_RECORDS = 500 -#: How long ONE run waits on a building snapshot before handing off to the next run (see 3 above). -BD_FILTER_WAIT = float(os.environ.get("AIOS_BD_FILTER_WAIT") or 120) -BD_FILTER_POLL = float(os.environ.get("AIOS_BD_FILTER_POLL") or 15) - -#: The operator set, enumerated BY REJECTION — send a bogus one and the API's own validation error -#: lists the legal set. The cheapest kind of measurement, and it makes this list a fact. -BD_FILTER_OPS = ("=", "!=", "<", "<=", ">", ">=", "in", "not_in", "includes", "not_includes", - "array_includes", "not_array_includes", "is_null", "is_not_null") -#: Operators that take NO value (everything else requires one). -BD_NULLARY_OPS = ("is_null", "is_not_null") - -#: Fields a predicate may name. MEASURED-accepted (24 of 25 probed; **`country_code` is REJECTED** -#: by the API and is therefore absent rather than offered-and-broken). -#: ⛔ `email_address`, `phone_number` and `business_email` ARE filterable and are DELIBERATELY -#: MISSING. Selecting people BY CONTACT DETAIL across 620M records is the materially heavier -#: privacy posture D-24 flags — a different licence question from per-URL enrichment, and one the -#: owner has not been asked. A vocabulary that cannot express it cannot be asked for it by -#: accident; adding them back is a decision, not a typo. -BD_FILTER_FIELDS = ("followers", "following", "posts_count", "avg_engagement", "biography", - "category_name", "business_category_name", "is_business_account", - "is_professional_account", "is_verified", "account", "full_name", - "external_url", "bio_hashtags", "post_hashtags", "profile_url", - "related_accounts", "profile_name", "id", "fbid", "highlights_count") -#: ⭐ The three the FIND SURFACE leads with, and the reason is measurement rather than taste: -#: these are the fields seen carrying values on real CORPUS rows. `category_name` and -#: `related_accounts` are filterable and were null/empty on every row we have looked at — a filter -#: on an unpopulated field returns nothing and looks exactly like "no such influencers exist". -BD_FILTER_LEAD = ("followers", "biography", "avg_engagement") - -#: ⭐⭐ WAVE 32 · T46 / DEBT D-167 — THE FILTER VOCABULARY HAS A PLATFORM NOW, AND IT COST MONEY -#: NOT TO. `BD_FILTER_FIELDS` above was measured against INSTAGRAM in wave 22; waves 29 and 30 hung -#: a second platform on the same tuple, so a `discover_tiktok` could be built on any of the **16 -#: fields Bright Data's TikTok Profiles dataset does not have** — and `BD_FILTER_LEAD`'s third -#: field, `avg_engagement`, is one of the three the Find panel LEADS with and is absent there. -#: A search on a field the corpus does not carry does not error: it returns nothing, and reads -#: exactly like *"no such creators exist"* after the money has been spent. -#: -#: ⛔ MEASURED, NOT REASONED — W30-B20, against TikTok's 40 declared dataset fields: **5 of the 21 -#: names survive.** They are these. Anything else is refused at the door with the field named, -#: which is `clean_predicates`' existing posture (*"refuses rather than coerces … the alternative -#: turns 'find me verified accounts in Georgia' into 'find me any account' and bills for the -#: difference"*) — now applied per platform rather than per product. -#: ⚠ NOT DERIVED FROM `connectors_tt.normalize_profile`, TEMPTING AS THAT IS. That mapper declares -#: the vendor's READ names on a returned row; these are the names its FILTER validator accepts, and -#: the two vocabularies are not the same thing on either platform (`awg_engagement_rate` reads, -#: `avg_engagement` filters). Deriving one from the other would look rigorous and be wrong. -TT_FILTER_FIELDS = ("followers", "following", "biography", "is_verified", "id") -#: The two of `BD_FILTER_LEAD`'s three that TikTok actually carries. `avg_engagement` is dropped -#: for the reason above — leading with a field the corpus cannot answer is worse than leading with -#: two. -TT_FILTER_LEAD = ("followers", "biography") - - -def filter_fields(kind=""): - """The searchable field names for one discovery kind — `(fields, lead)`. - - ⛔ ONE ACCESSOR, so the ROUTE that publishes the vocabulary and the VALIDATOR that enforces it - cannot come to disagree about it. That divergence is D-167 itself: the route served 21 names - with no platform dimension and `clean_predicates` validated against the same tuple with no kind - parameter, so both halves were consistently wrong together and nothing could notice. - """ - if str(kind or "") == "discover_tiktok": - return TT_FILTER_FIELDS, TT_FILTER_LEAD - return BD_FILTER_FIELDS, BD_FILTER_LEAD - -#: ⛔ C4 (wave 22) — THE TOO-BIG GUARD, and its unit is the PREDICATE, not the row count. -#: MEASURED (2026-08-04): `records_limit: 10` over a bare `followers > 10000` was still -#: `building` 50 minutes later — scan time tracks PREDICATE BREADTH; bounding the rows does not -#: bound the scan. What actually narrows a 620M-row corpus is CONTENT: a text/array match on a -#: field that discriminates. Numeric ranges and booleans partition the corpus into slabs the -#: scanner still has to walk, so they refine a search and cannot BE one. -BD_NARROWING_FIELDS = frozenset({ - "biography", "account", "full_name", "profile_name", "category_name", - "business_category_name", "external_url", "bio_hashtags", "post_hashtags", - "related_accounts", "id", "fbid", "profile_url"}) -#: The ops that actually pin content — `!=`/`not_includes` on a text field matches nearly the -#: whole corpus, which is breadth wearing a condition's clothes. -BD_NARROWING_OPS = frozenset({"=", "in", "includes", "array_includes"}) -#: Fields MEASURED carrying values on real corpus rows — D-25's fill-rate table, 50 mixed -#: profiles (snapshot `snap_msfrlbmt7qvej8uby`, collected 2026-08-05). The bar is ≥60% -#: populated: below it a filter misses more corpus than it matches, and the surface should -#: say so. MEASURED AND EXCLUDED: category_name 48%, business_category_name 34%, -#: related_accounts 22%, post_hashtags 18%, bio_hashtags 10% — and the PII trio the map never -#: carries anyway read 2%/0%/0%, so even the thing R3 forbids would barely have worked. -#: ⚠ posts_count was 100% populated on CORPUS rows — the fabricated-zero finding -#: (`_bd_posts_count`) is a SCRAPE-path fact and both stay true. -BD_POPULATED_FIELDS = frozenset({ - "account", "followers", "following", "posts_count", "avg_engagement", "biography", - "external_url", "is_business_account", "is_professional_account", "is_verified", - "full_name", "profile_name", "highlights_count", "id", "fbid", "profile_url"}) -BD_MIN_NARROWING = 1 - -# ── THE FIELD SCHEMA — what a person sees, and what they are allowed to ask. ────────────────── -# ⛔ THE SURFACE USED TO SHOW THE VENDOR'S OWN COLUMN NAMES AND ALL FOURTEEN OPERATORS ON EVERY -# ROW. So `is_business_account` offered `>=`, `fbid` sat in the list with no explanation of what -# it is, and `not_array_includes` was a thing a customer was expected to reason about. Owner: -# *"look at each damn schema and only provide toggles operator that make sense"*. -# -# ⚠ THE LABEL IS NOW LOAD-BEARING IN BOTH DIRECTIONS. The old comment in `AutomationFind.tsx` -# defended raw names on the grounds that a refusal names the field exactly as the row does — and -# it was right, which is why `field_label()` is used by the REFUSALS too. Prettifying only the UI -# is how you get an error message about `bio_hashtags` on a row labelled "Hashtags in bio". -BD_FIELD_LABELS = { - "followers": "Followers", "following": "Following", "posts_count": "Post count", - "avg_engagement": "Engagement rate", "biography": "Bio", "category_name": "Category", - "business_category_name": "Business category", "is_business_account": "Business account", - "is_professional_account": "Professional account", "is_verified": "Verified", - "account": "Handle", "full_name": "Name", "profile_name": "Profile name", - "external_url": "Link in bio", "profile_url": "Profile link", - "bio_hashtags": "Hashtags in bio", "post_hashtags": "Hashtags in posts", - "related_accounts": "Related accounts", "id": "Instagram ID", "fbid": "Facebook ID", - "highlights_count": "Story highlight count", -} -#: A one-line "what IS this" for the rows nobody can be expected to guess. Absent = self-evident; -#: a hint on all 21 rows is a hint on none of them (the wave-24 chip lesson). -BD_FIELD_HINTS = { - "account": "The @username", - "avg_engagement": "Likes and comments as a share of followers", - "id": "Instagram's internal number for the account. For matching a list you already have", - "fbid": "The linked Facebook id. For matching a list you already have", - "profile_name": "The name shown above the bio, when it differs from the account name", - "related_accounts": "Accounts Instagram suggests alongside this one", -} -#: The KIND decides the comparisons offered and the control drawn for the value. -BD_FIELD_KINDS = { - **{n: "number" for n in ("followers", "following", "posts_count", "avg_engagement", - "highlights_count")}, - **{n: "boolean" for n in ("is_business_account", "is_professional_account", "is_verified")}, - **{n: "tags" for n in ("bio_hashtags", "post_hashtags", "related_accounts")}, - **{n: "choice" for n in ("category_name", "business_category_name")}, - **{n: "text" for n in ("biography", "account", "full_name", "profile_name", "external_url", - "profile_url", "id", "fbid")}, -} -#: ⛔ ONLY WHAT MAKES SENSE, and the ORDER is the order they are offered in — the first entry of -#: each list is what `default_operator` must agree with. -BD_OPS_BY_KIND = { - # ⚠ `in`/`not_in` are DELIBERATELY ABSENT. "is any of" is what `=` becomes the moment a second - # value is typed (see `BD_MULTI_VALUE_OPS`), so offering both would be two controls for one - # idea — and the second one is the one written in vendor grammar. - "text": ("includes", "not_includes", "=", "is_not_null", "is_null"), - "choice": ("=", "!=", "is_not_null", "is_null"), - "number": (">=", "<=", "=", ">", "<"), - # A yes/no field has exactly one sensible comparison and a two-option value. `!=` on a boolean - # is `=` with the other value, written the confusing way. - "boolean": ("=",), - "tags": ("array_includes", "not_array_includes", "is_not_null", "is_null"), -} -#: Plain English for every comparison the surface can offer. The vendor's token stays on the wire -#: (it is what the API accepts); nobody has to read it. -BD_OP_LABELS = { - "includes": "contains", "not_includes": "does not contain", - "=": "is", "!=": "is not", "in": "is any of", "not_in": "is none of", - ">=": "at least", "<=": "at most", ">": "more than", "<": "less than", - "array_includes": "has any of", "not_array_includes": "does not have", - "is_null": "is empty", "is_not_null": "is not empty", -} -#: Yes/No, so a boolean is a two-item dropdown instead of a box you type `true` into. -BD_BOOLEAN_OPTIONS = [{"value": "true", "label": "Yes"}, {"value": "false", "label": "No"}] - -#: ⭐ THE OPERATORS THAT MAY CARRY SEVERAL VALUES — owner item 3, "if i want to include many -#: keywords like floral, flower, beauty". One condition row holds the list; `bd_filter_start` -#: expands it into a NESTED OR group, so it composes with the other conditions instead of forcing -#: the whole search to "match any" (which would also drag Followers into the union — the hole the -#: OR guard closed the same day). -#: -#: ⛔ POSITIVE OPERATORS ONLY, and this is a correctness line rather than a scope line. "does not -#: contain floral OR does not contain flower" matches nearly every account alive — a negative over -#: a list is an AND (De Morgan), and quietly OR-ing it would build a filter that reads like a -#: narrowing and behaves like the whole corpus. Negatives stay single-valued until someone needs -#: them enough to write the AND branch. -BD_MULTI_VALUE_OPS = frozenset({"includes", "=", "array_includes"}) -MAX_PREDICATE_VALUES = 12 - - -def field_label(name): - """The human name for a searchable field — used by the SURFACE and by every REFUSAL.""" - return BD_FIELD_LABELS.get(name) or str(name or "that field") - - -def field_kind(name): - return BD_FIELD_KINDS.get(name, "text") - - -def ops_for(name): - """The comparisons this field may be asked. A tuple, in offer order.""" - return BD_OPS_BY_KIND.get(field_kind(name), BD_OPS_BY_KIND["text"]) - -#: ⭐ WHAT A **FRESH** CONDITION ON A FIELD STARTS AS — and it lives HERE, beside the narrowing -#: law, because the two drifted apart and the drift cost a user their first automation. -#: -#: ⛔ THE SCAR, IN THREE WAVES. Wave 21 fixed a seeded condition the server refused for having no -#: value by making a new condition NULLARY (`is_not_null` — "this field has any value"), and its -#: comment called that "narrowing-in-the-right-direction". Wave 22 then wrote `BD_NARROWING_OPS` -#: and `is_not_null` is NOT IN IT, which silently made that default a condition the save door -#: always refuses. Wave 24 deleted the create wizard, so the Find panel became the ONLY way in — -#: and the refusal moved from a corner case to the first thing a new user meets. Three waves, one -#: sentence of drift, and nothing red anywhere at any point. -#: -#: So the rule is now DERIVED and ASSERTED (`verify_automation.py`): a fresh condition on a field -#: that CAN narrow must narrow. The client asks for `defaultOperator` per field and never picks -#: one itself — a client-side default is a second copy of this table, free to disagree with the -#: guard that judges it, which is exactly what happened. -BD_ARRAY_FIELDS = frozenset({"bio_hashtags", "post_hashtags", "related_accounts"}) -BD_BOOLEAN_FIELDS = frozenset({"is_business_account", "is_professional_account", "is_verified"}) -#: Narrowing fields whose values are IDENTIFIERS — a substring match on a handle or an id is a -#: worse question than an equality, and both narrow. -BD_EXACT_FIELDS = frozenset({"account", "id", "fbid", "profile_url"}) - - -def default_operator(name): - """The operator a NEW condition on `name` starts with. - - Narrowing wherever the field can narrow, so a fresh condition is ONE step (type a value) from - saveable rather than two (change the comparison, then type a value) — with the second step - being one the surface never told anyone to take. - """ - kind = field_kind(name) - if kind == "tags": - return "array_includes" - if kind in ("boolean", "choice"): - # ⚠ `choice` LANDS HERE AND NOT ON `includes`, which is the arm it used to fall through - # to (Category is a narrowing field). `includes` is not in `BD_OPS_BY_KIND["choice"]`, so - # the default would have been an operator the same module refuses to offer — the exact - # default-versus-guard split this function was written to close, reintroduced one commit - # later by adding a kind. Asserted per field in `verify_automation.py`. - return "=" - if kind == "number": - # `>=` cannot narrow the scan and is not pretending to — it is the comparison a person - # reaching for Followers means, and the guard's sentence is the honest answer when it is - # the ONLY condition present. - return ">=" - return "=" if name in BD_EXACT_FIELDS else "includes" - - -def predicate_narrows(p): - """Does ONE predicate narrow the corpus scan? Content field + content op, nothing else.""" - return (str((p or {}).get("name") or "") in BD_NARROWING_FIELDS - and str((p or {}).get("operator") or "") in BD_NARROWING_OPS) - - -def narrowing_refusal(preds, operator="and", kind=""): - """C4's server law: the sentence when a predicate set does not narrow, else ''. One function - so create/patch (via `clean_predicates`) and RUN (legacy stored configs predate the law) - refuse in the same words. - - ⭐ THE JOIN IS PART OF THE LAW, and it was missed for two waves. The guard asked "does ANY - predicate narrow?" — a question that is only correct under AND. Under **OR** the result is a - UNION, so the query is exactly as broad as its WIDEST branch: `biography includes "florist" OR - followers >= 10000` saved clean and asked the vendor for every account over ten thousand - followers, which is the measured 50-minute / NOT_ENOUGH_FUNDS shape the guard exists to stop. - A money wall that a dropdown three rows above it can walk through is not a wall. - - So: under OR every branch must narrow; under AND one is enough. An EMPTY set never narrows - either way — `all([])` is True, which would have made "no conditions at all" the widest legal - query of the lot. - """ - preds = list(preds or []) - if str(operator or "").lower() == "or": - if preds and all(predicate_narrows(p) for p in preds): - return "" - return ("with Match set to any, every condition has to describe the account itself " - "(Bio, Handle, Name, a hashtag). Matching any means the results are added " - "together, so one follower or yes/no condition widens the whole search. Narrow " - "every condition, or set Match to all") - if any(predicate_narrows(p) for p in preds): - return "" - # ⭐ WAVE 32 · T46 (D-167's neighbour) — THE NETWORK'S OWN NAME. This sentence said - # "Instagram" to somebody building a TIKTOK search, on the refusal they are most likely to - # read, because the string predates the second platform. `discovery_facts` already carries the - # network's name for exactly this class of string, so there is no second literal. - _net = discovery_facts(kind)[0] if kind else PLATFORM_INSTAGRAM - return ("add a condition that describes the account itself. Bio contains, Handle is, a " - f"hashtag. Follower counts and yes/no conditions alone match too much of {_net} " - "to search") - - -def filter_meta(): - """C4's per-field flags for the Find surface: `[{name, populated, narrowing, defaultOperator}]` - over the same 21 names `BD_FILTER_FIELDS` offers (the 3 PII fields stay structurally absent, - R3). - - `defaultOperator` rides here rather than being a client constant for the reason written over - `default_operator`: the client's own default was `is_not_null` for every field, which the - narrowing guard refuses on every field. - """ - return [{"name": n, - "label": field_label(n), - "hint": BD_FIELD_HINTS.get(n, ""), - "kind": field_kind(n), - "populated": n in BD_POPULATED_FIELDS, - "narrowing": n in BD_NARROWING_FIELDS, - "defaultOperator": default_operator(n), - # The comparisons THIS field may be asked, already in plain English and already in - # offer order. A client that filtered a global list by field kind would be a second - # copy of `BD_OPS_BY_KIND`, and the copy is what goes stale. - "operators": [{"value": o, "label": BD_OP_LABELS.get(o, o), - "nullary": o in BD_NULLARY_OPS, - # `in`/`is any of` and the tag operators take a LIST — the control - # has to know that, and deriving it from the token in the client is - # the same second-copy mistake one level down. - "multi": o in BD_MULTI_VALUE_OPS} - for o in ops_for(n)], - "options": BD_BOOLEAN_OPTIONS if field_kind(n) == "boolean" else []} - for n in BD_FILTER_FIELDS] - -#: ⭐ WAVE 25 · CONTRACT C1 — THE UNIFIED, CROSS-TENANT INSTAGRAM PRESET SET. -#: -#: ONE server-owned list. Every tenant gets the same keys and the same labels, which is what -#: "unified across tenant" means and what makes the pooled master series joinable at all — two -#: tenants calling the same number `followers` and `follower_count` is a pool you cannot query. -#: ⛔ THERE IS NO CLIENT COPY OF THIS LIST. C and D read it off the wire (`GET -#: /automations/presets`); a client-side copy is the wave-9 silent-drop failure in a new hat. -#: -#: ⭐ R3 — THESE HOLD **LATEST**, AND NOTHING ELSE. The full time series stays as append rows in -#: `ut_ig_snapshots`. ONE STORE FOR ONE SERIES: no per-record `json` history column, no second copy -#: of a number that already has a home. -#: ⛔ W29-T34 — THIS LINE USED TO ADD *"which the metric fields already read"*, AND THAT WAS WRONG. -#: A `metric` field reads the PLATFORM-WIDE MASTER, not this tenant's snapshot table: -#: `compute_metric_cells` imports `ig_master` and calls `ig_master.series_for(...)` — a -#: cross-tenant pooled repo in a different HF dataset, which is the whole point of C6's pooling and -#: which no rollup kind can reach. The metric header below says exactly that, so the module was -#: contradicting itself on which store answers the question. -#: ⚠ And the practical finding behind the correction, recorded because it is surprising: a live -#: census of all four tenants (13 tables / 228 fields / 64 store files) found **ZERO** metric fields -#: in existence, while 47 rollups are live. The vocabulary stays by owner ruling; nothing uses it. -#: -#: ⭐⭐ 2026-08-07 — THIS IS NOW *EVERY* PROFILE FIELD THE VENDOR RETURNS, BY OWNER INSTRUCTION, -#: AND THAT REVERSES D-25's FILL-RATE RULE. Owner, verbatim: *"make sure to have ALL Fields -#: available to us from Bright Data to be pre-set Fields for us and populated."* -#: -#: ⛔ THE RULE IT REPLACES IS WRITTEN DOWN HERE RATHER THAN DELETED, because it was a good rule -#: and the next reader will otherwise re-derive it and prune these columns back. It said: a -#: LATEST-value column costs something a snapshot row does not — it is a column every user sees on -#: their grid forever — so the split followed D-25's MEASURED fill-rates (50 mixed profiles, -#: snapshot `snap_msfrlbmt7qvej8uby`) and a field earned a column only at ≥60% populated. That is -#: why `business_category` (34%), `bio_hashtags` (10%), `post_hashtags` (18%) and `pronouns` were -#: captured into the snapshots and were NOT preset columns. -#: -#: ⚠ THE MEASUREMENT IS UNCHANGED AND STILL WORTH KNOWING — several of these columns WILL be -#: mostly blank on the scrape path, and D-25's central finding is why: **the scrape path and the -#: corpus path are different data.** `avg_engagement` is 0% populated on scrape and 88% on corpus; -#: `category` is 0% on scrape and 70% on corpus. So a blank here is very often "this ROUTE does -#: not carry it", not "this account does not have it" — which is exactly what `enriched_at` and -#: the blank-never-zero law exist to keep honest. What changed is the ANSWER to "does a -#: sometimes-blank column earn a place on the grid", and that was always the owner's call to make. -#: Source data retains every field the provider returns. The only separate decision is whether to -#: request the paid full Comments dataset (`commentMetrics`, default OFF). -#: -#: ⭐ WAVE 26 · R5 — THE NETWORK VOCABULARY. A handle is only unique WITHIN a network, so this is -#: half of the candidate identity (contract C3: the upsert key is `(platform, handle)`). -#: -#: ⚠ THE VALUES ARE DISPLAY STRINGS ON PURPOSE. They land in an ordinary `text` cell that a person -#: reads, filters and groups by, so `"Instagram"` beats `"ig"` — and the set is small, closed and -#: written down here rather than inferred from a runner's module name, because the day a second -#: runner spells it `"instagram"` is the day the dedup key stops working and nothing errors: the -#: profile simply appears twice. -#: ⭐ 2026-08-07 — WHERE A PROFILE HANDLE POINTS, as `core.user_tables.PROFILE_SOURCES` spells it. -#: A local literal for the same boot-path reason `MACHINE_OWNERS` and `UT_FIELD_TYPES` are locals -#: here, and held in step by a gate rather than an import. ⛔ NOT the same vocabulary as -#: `PLATFORM_*` below: this names the FLAG's source (which validator reads the cell), that names -#: the NETWORK a row belongs to (half of the identity). They read alike and mean different things, -#: which is exactly why both are written down instead of inferred. -PROFILE_SOURCE_IG = "instagram" -#: ⚠ Its TikTok twin, `PROFILE_SOURCE_TT`, is declared UP with the `ut_tt_*` schemas (wave 29): the -#: field lists there are evaluated at import and would not see a constant defined here. - -PLATFORM_INSTAGRAM = "Instagram" -#: ⭐ WAVE 29 — the `ut_tt_*` family stamps this on every row of all five of its tables. (It really -#: was written by nothing until D-9 landed; the note that said so is kept in the log, not here.) -PLATFORM_TIKTOK = "TikTok" -PLATFORM_FACEBOOK = "Facebook" -PLATFORMS = (PLATFORM_INSTAGRAM, PLATFORM_TIKTOK, PLATFORM_FACEBOOK) - - -def discovery_facts(kind): - """⭐⭐ WAVE 30 · T05 — THE THREE THINGS THAT GENUINELY DIFFER BETWEEN THE DISCOVERY KINDS: - the network's NAME, the database a run writes to when the config names none, and that - database's label. `(platform, table, label)`. - - ⛔ WHY A FUNCTION AND NOT A DICT LITERAL: `DISCOVER_TABLE` and `DISCOVER_LABEL` are declared - several hundred lines BELOW this point, so a dict evaluated here would NameError at import. - A function body resolves at call time and does not care. (`PROFILE_SOURCE_IG`'s note directly - above records the same import-order trap costing a constant its natural home.) - - ⛔ AND WHY IT EXISTS AT ALL. These three facts were written out inline at FIVE call sites — - `clean_config`, `graph`'s discovery arm, `graph`'s `det` dict, `_flow_table` and - `compose_sentence` — and wave 29 taught four of the five about TikTok by not touching them: - they tested the string `"discover_instagram"`, so a stored TikTok automation fell through to - a default meant for Instagram, or to no arm at all. Every one of those misses was SILENT and - every gate stayed green. `DISCOVERY_KINDS` answers *"is this a corpus search?"*; this answers - *"whose?"* — and between them a sixth platform is one tuple entry and one arm here, rather - than a hunt through the file for string comparisons somebody has to think to look for. - - ⚠ DELIBERATELY NOT A PLATFORM REGISTRY. The dataset ids, the field maps and the row mappers - stay in the connector modules that own them. This is the presentation-and-default quartet the - ENGINE needs, and widening it is how it becomes a second `SOURCES` with a longer name. - - ⭐ T06 ADDED THE FOURTH ELEMENT, `spawn_fields` — the field list `_presets_after_write` gives - the target database on SAVE. It belongs here rather than beside that function because it must - equal what the RUN-TIME `ut_ensure` uses, and wave 25's R10 exists precisely to stop those two - disagreeing: columns that change between Save and the first run are two different answers to - "what does this database look like". - ⚠ THE TWO PLATFORMS PASS DIFFERENT-LOOKING LISTS AND THAT IS CORRECT, not an oversight. - Instagram spawns `CANDIDATE_FIELDS` (= the preset set PLUS the discovery bookkeeping) because - its preset list alone is a subset of what its runner writes; TikTok's `TT_PROFILE_FIELDS` - already carries its own bookkeeping columns, so it IS the whole set. Each side is the list its - own runner ensures — which is the rule, rather than "both use the one named CANDIDATE". - """ - if kind == "discover_tiktok": - return (PLATFORM_TIKTOK, TT_PROFILE_TABLE, TT_TABLE_LABELS[TT_PROFILE_TABLE], - TT_PROFILE_FIELDS) - return PLATFORM_INSTAGRAM, DISCOVER_TABLE, DISCOVER_LABEL, CANDIDATE_FIELDS - -#: ⭐ WAVE 26 · R3 — THE TYPES ARE HONEST NOW, AND THE MIGRATION IS THE PRICE OF THAT. -#: -#: This block used to say the `text` types were deliberate, and the reasoning was sound as far as -#: it went: `ut_ensure` merges by KEY and never re-types an existing column, so re-declaring -#: `followers` as `int` gives NEW tables a schema every table already in production does not have -#: — one vocabulary with two shapes, the exact drift this list exists to prevent. That argument -#: was never wrong; it was an argument for doing the MIGRATION, and the note used it as a reason -#: not to. Owner, 2026-08-06: *"Why is everything that instagram field found is in a text format? -#: change this."* -#: ⛔ SO THE TWO HALVES ARE INSEPARABLE AND SHIP TOGETHER. Declaring a type here without -#: `migrate_ig_field_types()` reintroduces exactly the split-schema the old note feared — and it -#: would do it silently, because a merged-by-key field list produces no error when two tables -#: disagree about what a column IS. -#: -#: ⚠ `avg_engagement` CARRIES A UNIT CONVERSION, NOT JUST A TYPE (amendment C1-a). The vendor -#: sends a 0–1 fraction (MEASURED: 0.0074 / 0.0656 / 0.0014 / 0.0148 / 0.0274 / 0.0091) and this -#: product's `pct` renders the stored number with a `%` appended — so storing the raw fraction -#: would print every creator in the book as `0.0%`. It is stored ×100 from here on, at the -#: mapping and in the migration both. See `_pct100`. -PRESET_PROFILE_FIELDS = [ - # ⭐ WAVE 26 · R5 — WHICH NETWORK THIS HANDLE IS ON, and it is half of the identity now. - # `@inayma` on Instagram and `@inayma` on TikTok are two accounts owned by two different - # people as often as not, so the dedup key is (platform, handle) and never handle alone - # (C3). ⛔ NOT called `source`: `SNAPSHOT_FIELDS` already spends that word on *which rung - # answered* (`field_def("source", "Read via")`), and one word meaning two things across two - # tables in one product is the drift this whole list exists to prevent. - field_def("platform", "Platform"), - # ⭐⭐ 2026-08-07 (owner ruling) — THE HANDLE IS THE PRIMARY COLUMN *AND* THE ENRICH BINDING. - # Owner: *"if a user use this automation for instagram scraping, the Unique ID will always be - # REPLACED or made as the Handle. in any case the user can add their own record right and edit - # those records."* Two declarations, one field, and they are the same sentence read twice: - # - # `pinned` — the grid's identity column is `find(f => f.pinned) ?? fields[0]`, so WITHOUT - # this the primary is an accident of creation order. It is a DECLARATION, not a - # reorder: the stored field order is left alone (saved view orders keep working) - # and the client pins the column to position 0 wherever it sits. That is why this - # fixes tables that already exist without moving a single stored field. - # `profile` — C3/R7's flag. It makes `handle` the column `enrich_instagram` BINDS to - # (`profile_field_key` step 2), which is what turns "I added a profile to my - # database, how do I enrich it?" from an unanswerable question into a step. It - # also validates the cell (`@Name`, a pasted instagram.com link and a bare handle - # all normalise to the bare handle) and routes a typed value to the SHARED - # definition row rather than one user's overlay (C3-A1) — which is precisely what - # makes the owner's second clause true: a record you add BY HAND is a record the - # engine can read and enrich. - # - # ⛔ THE TWO KEYS MUST SURVIVE `_clean_field` UNCHANGED or `verify_api` W25-1 goes red. The flag - # is also why this field can never be retyped: `_clean_field` REFUSES a profile flag on - # anything but `text`. - field_def("handle", "Handle", pinned=True, profile={"source": PROFILE_SOURCE_IG}), - field_def("profile_url", "Profile", "url"), - field_def("full_name", "Name"), field_def("followers", "Followers", "int"), - field_def("following", "Following", "int"), - field_def("avg_engagement", "Avg engagement", "pct"), - field_def("bio", "Bio"), field_def("external_url", "Link in bio", "url"), - field_def("verified", "Verified", "checkbox"), field_def("category", "Category"), - # --- the enrichment half: what `run_field_instagram` already pulls, kept to the fields - # MEASURED ≥60% populated (see the note above). - field_def("posts_count", "Post count", "int"), - field_def("highlights_count", "Story highlight count", "int"), - field_def("is_business", "Business account", "checkbox"), - field_def("is_professional", "Professional account", "checkbox"), - # ⚠ STAYS `text`. It is a 17-digit opaque identifier, not a quantity — typing it `int` would - # invite a grid to sum it, and some of them exceed 2^53 so a JS reader would round it. - field_def("ig_id", "Instagram id"), - # --- ⭐ 2026-08-07: the rest of the vendor's profile schema, promoted by owner instruction - # (see the block comment above for what that reverses). Types are HONEST from birth, which - # costs nothing here: no table has ever carried these columns, so there is no stored `text` - # definition for the migration to convert — R3's conversion rule applies to the columns that - # already exist, and these are new everywhere. - field_def("business_category", "Business category"), - field_def("is_private", "Private account", "checkbox"), - field_def("bio_hashtags", "Bio hashtags"), - field_def("pronouns", "Pronouns"), - field_def("profile_name", "Profile name"), - field_def("is_joined_recently", "Joined recently", "checkbox"), - field_def("has_channel", "Has channel", "checkbox"), - field_def("partner_id", "Partner id"), - field_def("external_url_title", "Link title"), - # ⚠ `text` for `ig_id`'s reason, one identifier over: it is a name, not a quantity. - field_def("fbid", "Facebook id"), - field_def("related_accounts", "Related accounts"), - field_def("country_code", "Country"), - field_def("source_payload", "Source data", "json"), - # ⭐⭐ 2026-08-07 (owner instruction) — THE RELATION AND THE ROLLUPS OVER IT. - # - # Owner: *"an enrichment automation should spawn relevant Post/Comment database that is - # linked to the profile automatically… and this rollup needs to have formula that we can use - # to calculate things like average Views over last N posts."* Both halves are these five - # columns, and they are PRESETS rather than something a person assembles, because "linked - # automatically" is the instruction — a relation you have to wire up by hand is the feature - # not existing. - # - # ⛔ `from` IS NOT DECLARED, ON PURPOSE. `_link_from_key` falls back to the PROFILE-flagged - # column and then the pinned one, both of which are `handle` on every preset table — so this - # links correctly on a database where the handle column was renamed, and on one where the - # flag sits somewhere unexpected. Naming `handle` here would be the hard-coded subject - # [[gate-answers-the-wrong-question]] warns about, one layer down. - # ⚠ "Post rows", NOT "Post count" — one opens records and one reports the account total. - # Two count-like columns with the same label would be unreadable. Caught by the label-collision - # gate rather than on screen, which is what that gate is for. - field_def("posts_link", "Post rows", "link", - description="Post records linked to this profile.", - link={"table": IG_POSTS_TABLE, "on": "influencer_key"}), - field_def("profile_snapshots_link", "Profile history", "link", - description="Profile snapshots linked to this profile.", - link={"table": IG_SNAPSHOTS_TABLE, "on": "influencer_key"}), - field_def("post_snapshots_link", "Post measurement rows", "link", - description="Post engagement measurements linked to this profile.", - link={"table": IG_POST_SNAPSHOTS_TABLE, "on": "influencer_key"}), - field_def("comments_link", "Comment rows", "link", - description="Comment records linked to this profile.", - link={"table": IG_COMMENTS_TABLE, "on": "influencer_key"}), - # ⚠ THE ROLLUPS READ `ut_ig_posts`' OWN LATEST COLUMNS, which is why those exist — a rollup - # is ONE HOP (Airtable's rule and ours), and the engagement SERIES lives one table further - # out in `ut_ig_post_snapshots`. - # ⚠ 12 IS THIS MEASURE'S OWN WINDOW (`AVG_WINDOW_POSTS`), NOT THE CAPTURE CAP. It used to be - # `MAX_POSTS_PER_PULL` and read "the vendor's ceiling" — both halves are now wrong: the cap is 30 - # (owner, 2026-08-09) and 30 is not the vendor's limit either. Binding these four to the cap meant - # raising it silently redefined every column named `_12`. `sortBy` is mandatory alongside a - # `limit`, and this is why — "the last 12" has to name what makes one post later than another. - # ⭐ UN-RETIRED 2026-08-08 together with the column it averages. It was retired for four hours - # because its input was an account-grain constant; with the input now a real per-reel - # measurement the average means what its label says again. - field_def("avg_views_12", "Avg views · last 12 posts", "rollup", - rollup={"link": "posts_link", "field": "views", "fn": "average", - "limit": AVG_WINDOW_POSTS, "sortBy": "posted_at", "sortDir": "desc", - "distinctBy": "shortcode"}), - field_def("avg_plays_12", "Avg plays - last 12 posts", "rollup", - rollup={"link": "posts_link", "field": "plays", "fn": "average", - "limit": AVG_WINDOW_POSTS, "sortBy": "posted_at", "sortDir": "desc", - "distinctBy": "shortcode"}), - field_def("avg_likes_12", "Avg likes · last 12 posts", "rollup", - rollup={"link": "posts_link", "field": "likes", "fn": "average", - "limit": AVG_WINDOW_POSTS, "sortBy": "posted_at", "sortDir": "desc", - "distinctBy": "shortcode"}), - field_def("avg_comments_12", "Avg comments · last 12 posts", "rollup", - rollup={"link": "posts_link", "field": "comments", "fn": "average", - "limit": AVG_WINDOW_POSTS, "sortBy": "posted_at", "sortDir": "desc", - "distinctBy": "shortcode"}), - # ⭐ THE ONE HONEST POST COUNT WE HAVE. D-82: the vendor's `posts_count` is a FABRICATED ZERO - # on the paid rung (49/49 rows measured), so it is discarded and that column reads blank after - # a paid enrich. This counts the post rows actually captured — a different number and a true - # one, which is why it gets its own column and its own label rather than quietly filling - # `posts_count` with something that is not what that field means. - field_def("posts_captured", "Posts captured", "rollup", - rollup={"link": "posts_link", "fn": "countall", "distinctBy": "shortcode"}), - field_def("profile_reads", "Profile reads", "rollup", - rollup={"link": "profile_snapshots_link", "fn": "countall", - "distinctBy": "snapshot_key"}), - field_def("post_measurements_captured", "Post measurements captured", "rollup", - rollup={"link": "post_snapshots_link", "fn": "countall", - "distinctBy": "post_snapshot_key"}), - field_def("comments_captured", "Comments captured", "rollup", - rollup={"link": "comments_link", "fn": "countall", - "distinctBy": "comment_key"}), - # ⭐⭐ WAVE 27 ITEM 16 — WHERE THIS CREATOR ACTUALLY IS, DERIVED, FOR NOTHING. - # - # ⛔ THE VENDOR DOES NOT SELL THIS. `country_code` was MEASURED `None` on every corpus row we - # have ever looked at, and the filter API REFUSES it as a predicate — so residency is the one - # thing a buyer most wants and the one thing the profile row cannot answer. The competitor - # teardown found the same gap solved by GUESSING from bio words, two buckets deep - # ([[janney-ai-teardown]]). - # - # ⭐ WE ARE ALREADY HOLDING THE ANSWER AND WERE THROWING IT AWAY. Each post carries the place - # it was tagged in, and the enrich has ALREADY BOUGHT twelve of them: `tagged_location` on the - # post row, plus the vendor's fuller `location`/`location_details` inside the paid - # `source_payload` that `_bd_tagged_location` normalises away. The modal city across a - # creator's own posts is a far better residency signal than a word in a bio, and it costs a - # dict comprehension. **Zero new vendor spend** — this is the whole reason it is a v1. - # - # ⚠ IT IS A GUESS AND THE COLUMN SAYS SO, IN ITS NAME AND IN ITS NEIGHBOUR. A travel creator - # posting from twelve cities gets a low confidence rather than a confident wrong answer, and - # `location_confidence` is the number a person filters on before trusting the guess. A single - # geotagged post produces NO guess at all — n=1 dressed as a pattern is what - # `SEED_MIN_ROWS_SHARING` refuses fifty lines up, and it would read as 100% certain. - field_def("location_guess", "Location (guess)"), - field_def("location_confidence", "Location confidence", "pct"), - # ⭐ R3's stamp. WITHOUT IT THE WHOLE SET IS UNREADABLE: a blank `followers` means "never - # enriched" and a stale one means "enriched in March", and no cell on the row can tell them - # apart. It is the single field that turns the other fifteen from numbers into measurements. - field_def("enriched_at", "Enriched at", "date"), - # ⭐ WAVE 26 · R1 — THE LAST-N POST WINDOW, AND IT IS A VIEW RATHER THAN A STORE. - # ⛔ READ THE R3 NOTE ABOVE BEFORE CHANGING THIS. "One store for one series" still holds: the - # authoritative post record is `ut_ig_posts` (keyed by shortcode, so it ACCUMULATES) and the - # authoritative engagement series is `ut_ig_post_snapshots` (append-per-pull, carrying - # views/likes/comments at a `pulled_at`). This cell is a DERIVED window over those two, - # rewritten each run, so a person reading the profile row can see the recent posts without a - # join — and deleting it would cost a convenience, never a measurement. Shape: contract C2. -] -#: The keys the preset set owns — derived, so a field added above cannot be forgotten here. -PRESET_PROFILE_KEYS = tuple(f["key"] for f in PRESET_PROFILE_FIELDS) - -#: ⭐ 2026-08-07 — WHICH preset field declares the primary column, and WHICH declares the profile -#: flag. DERIVED for the same reason `PRESET_PROFILE_KEYS` is, and the negative control is what -#: argued for it: with `"handle"` hard-coded in the migration, stripping the declaration left the -#: migration cheerfully stamping a column the product no longer claimed. Moving a declaration now -#: moves the migration with it, and REMOVING one stops the migration rather than leaving it to -#: enforce a rule nobody declares any more. -#: ⚠ Both are `""` when nothing declares them, and every reader treats `""` as "do nothing" — -#: never as "field number zero". -PRESET_PINNED_KEY = next((f["key"] for f in PRESET_PROFILE_FIELDS if f.get("pinned")), "") -PRESET_FLAG_KEY = next((f["key"] for f in PRESET_PROFILE_FIELDS if f.get("profile")), "") - -#: Discovery's table is the preset set PLUS its own bookkeeping. ⛔ DERIVED, NEVER RE-TYPED: the -#: alternative is two lists that describe the same columns and drift one label at a time, which is -#: the failure C1 exists to prevent. The five below are facts about the SEARCH (how often we found -#: them, who for, and a human's decision) rather than about the profile, so they are discovery's -#: and not part of the cross-tenant set. -CANDIDATE_FIELDS = [ - *PRESET_PROFILE_FIELDS, - # ⭐ WAVE 26 · R3 — `first_found` / `last_found` ARE DATES, not ISO strings in a text cell. - # They were written by `_iso()`, so the cell read `2026-08-05T14:03:11+07:00` and the owner - # called that format "extremely confusing" — correctly: it is a machine timestamp shown to a - # person, unsortable as a date and unfilterable by "last 7 days". The STAMP still carries its - # offset everywhere it is a time axis (`ut_ig_snapshots.pulled_at` is untouched); what - # changed is that "when did we first see this account" is a DAY, and a day is all anybody - # asks it for. `migrate_ig_field_types()` converts the stored strings. - field_def("found_count", "Times found", "int"), - field_def("first_found", "First found", "date"), - field_def("last_found", "Last found", "date"), - # ⛔ DEBT D-73 — THIS NOTE STATED A RETIRED LAW AS CURRENT FACT, directly above the field it - # describes, which is the first place anybody looks up what this column means. It read: *"the - # candidate pool is PER-USER — the upsert key is the COMPOUND (handle, created_by), so two - # people discovering the same profile each get their own row, their own found_count and their - # own review card."* Wave 26 · R4/R5 retired every clause of that. - # - # ⭐ THE CURRENT LAW: the identity is `(platform, handle)` and the TENANT is the unit. Two - # people in one workspace who discover the same profile share ONE row, one `found_count` and - # one history — because they are looking at one company's leads, not two private lists. - # `created_by` survives as an INFORMATIONAL stamp only: "Found by", answering who saw it - # first. It is no part of the dedup key, and a run must never branch on it. - # ⚠ Stamped by the RUNNER (the automation's creator), never by `_candidate_row` — unchanged, - # and the one clause of the old note that was still true. - field_def("created_by", "Found by"), - # ⛔ WAVE 26 · R6 — `tracked` WAS HERE AND IS DELETED. Owner, 2026-08-06: *"We already have - # a Field called stage to track the progress of the Automation per Record. We don't need - # another checkbox for this."* Correct, and it had been true since wave 22 shipped the stage - # column: every candidate carried BOTH a `stage_` select saying where it was AND a - # boolean saying whether it was kept — two progress fields that could disagree, with no rule - # about which one won. The stage field is the one progress column. Nothing replaces this. -] - - -def _profile_backlink_field(table_key, table_label, profile_key): - """A deterministic reciprocal link from one canonical IG table to one profile database. - - The key includes the target table identity, so ten separate Profile databases can all point - into the same canonical Posts/Comments/history tables without one relation overwriting the - next. The join is derived in both directions and therefore needs no cross-table fan-out write. - """ - digest = hashlib.sha1(str(table_key).encode("utf-8")).hexdigest()[:10] - return field_def( - f"profiles_{digest}", f"Profiles - {str(table_label or table_key)[:48]}", "link", - description=f"Profile records from {str(table_label or table_key)[:48]} linked to this row.", - link={"table": str(table_key), "on": str(profile_key), "from": "influencer_key"}, - ) - - -def _profile_schema_for(bound_key): - """Preset fields for an existing Profile table without inventing a second identity column.""" - out = [] - for field in PRESET_PROFILE_FIELDS: - item = dict(field) - if item.get("key") == PRESET_FLAG_KEY and bound_key != PRESET_FLAG_KEY: - item.pop("profile", None) - out.append(item) - return out - - -def _tt_profile_schema_for(bound_key): - """⭐ WAVE 30 · T08 — TikTok preset columns for a database whose profile column is `bound_key`. - - ⛔ NOT `_profile_schema_for` WITH A LIST ARGUMENT, and the difference is not stylistic. That - function walks `PRESET_PROFILE_FIELDS` and only ever removes the flag from `PRESET_FLAG_KEY`, - because on the Instagram side the flag lives on exactly one known column. TikTok's binding may - be ANY column a person named (`profile_field_key` step 1), so the rule here has to be stated - the other way round: **every field that is not the bound one arrives as DATA**, stripped of - both the identity flag and `pinned`. - ⚠ Otherwise a database whose TikTok column is `creator` would gain a rival `handle` carrying - `profile: {source: "tiktok"}` — a SECOND profile column, which `user_tables` refuses at both - write doors, so the whole top-up would be rejected and every preset cell would then be dropped - for want of a column. One shared helper for the save-time and run-time paths, so those two - cannot answer "which columns does a TikTok enrich need" differently. - """ - out = [] - for field in TT_PROFILE_FIELDS: - item = dict(field) - if str(item.get("key") or "") != str(bound_key or ""): - item.pop("profile", None) - item.pop("pinned", None) - out.append(item) - return out - - -def _locked_ig_field(field): - """One canonical Instagram field with the immutable preset declaration attached.""" - item = dict(field) - automation = dict(item.get("automation") or {}) - automation["preset"] = True - item["automation"] = automation - return item - - -def _profile_binding(table): - """The declared Profile identity field, falling back only to the canonical handle.""" - fields = list((table or {}).get("fields") or []) - flagged = next((str(f.get("key") or "") for f in fields - if isinstance(f.get("profile"), dict)), "") - if flagged: - return flagged - return PRESET_FLAG_KEY if any(f.get("key") == PRESET_FLAG_KEY for f in fields) else "" - - -def _ig_schema_contract(rt, profile_tables=None): - """The complete per-tenant Instagram graph contract, including every Profile backlink. - - The fixed child tables are shared. Profile tables are discovered structurally, so an older - user-named target and the built-in candidate database receive the same preset columns, Links, - Rollups, locks, and reciprocal fields. Retired machine keys are the only columns deleted. - """ - tables = ut_all(rt) - # ⭐ WAVE 32 · T42 — `_ig_contract_tables`, NOT `_ig_profile_tables`: this function decides - # which databases RECEIVE Instagram's 48 columns, and the owner's rule is that a database gets - # them only if it is used for Instagram. See that function for why the detector stays wider. - profile_keys = sorted(set(profile_tables if profile_tables is not None - else _ig_contract_tables(rt))) - backlinks, wanted, drops = [], {}, { - IG_SNAPSHOTS_TABLE: {"post_hashtags"}, - # ⚠ `views` IS DELIBERATELY ABSENT FROM THIS DROP SET AGAIN. It was dropped earlier today - # while it carried Bright Data's account-grain number; it is now declared in POST_FIELDS - # and fed by the `ig_post_views` capability. Leaving it here would have made the migration - # delete, on every authenticated read, the column the same release just added — the - # drop set and the field list are two halves of ONE contract and must move together. - } - for table_key in profile_keys: - table = tables.get(table_key) or {} - bound = _profile_binding(table) - if not bound: - continue - label = str(table.get("label") or table_key) - backlinks.append(_profile_backlink_field(table_key, label, bound)) - schema = _profile_schema_for(bound) - # ⭐⭐ 2026-08-10 (owner: *"retire the old hardcoded method and replace the Instagram - # database work with correct Rollup and Link fields"*) — THE DERIVED OVERLAY, AND IT - # BELONGS HERE RATHER THAN IN A SEPARATE MIGRATION. - # - # ⛔ A standalone converter would LOSE. `_reconcile_ig_graph_fields` overwrites every - # contract key it owns — `rollup` included, by its own note — so a column converted beside - # the contract is re-typed back to `int` on the next reconcile, silently, hours later. - # Making the CONTRACT itself say "this column is derived" leaves exactly one writer of the - # schema instead of two that disagree. - # - # ⚠ THE SET IS PER TENANT AND IS A PROOF, NOT A LIST. `_derived_profile_columns` converts a - # column only where this tenant's own rows show the fold already equals the stored value — - # measured, because a named list would have blanked 221 live cells on nurilab alone - # (`avg_engagement` and `category`, 62 each). A tenant whose series is thinner converts - # fewer columns and loses nothing; as its history fills in, later passes convert more. - derived = set(_derived_profile_columns(table, tables.get(IG_SNAPSHOTS_TABLE) or {})) - wanted[table_key] = [_as_derived_field(f) if str(f.get("key")) in derived else f - for f in schema] - # `tracked` was the pre-Stage progress checkbox. Keeping both allows two progress states - # to disagree, so it is retired by key just like the old nested Posts JSON. - drops[table_key] = {"posts", "post_hashtags", "tracked"} - wanted.update({ - IG_SNAPSHOTS_TABLE: [*SNAPSHOT_FIELDS, *backlinks], - IG_POSTS_TABLE: [*POST_FIELDS, *backlinks], - IG_POST_SNAPSHOTS_TABLE: [*POST_SNAPSHOT_FIELDS, *backlinks], - IG_COMMENTS_TABLE: [*COMMENT_FIELDS, *backlinks], - }) - return wanted, drops - - -# Only a unit-changing retype needs a cell conversion. Integer/checkbox/date cells already use -# the scalar strings their renderers expect; engagement is the exception because Bright Data's -# 0-1 fraction becomes this product's 0-100 percentage. -_IG_RETYPE_CONVERTERS = { - (IG_SNAPSHOTS_TABLE, "avg_engagement"): _pct100, -} - - -def _reconcile_ig_graph_fields(rt, wanted_by_table, drop_by_table=None): - """Repair machine-owned IG field declarations in one coalesced store update. - - `ut_ensure` deliberately merges only missing keys. That is correct for user columns but not - sufficient for a canonical relation: a stale `link.table`, rollup function, or type would - survive forever. This pass overwrites the contract keys for the fields we own, preserves - unrelated/user fields, and removes only explicitly retired preset columns plus their cells. - """ - drop_by_table = drop_by_table or {} - wanted_by_table = { - key: [_locked_ig_field(f) for f in fields] - for key, fields in wanted_by_table.items() - } - changed = False - - def _up(cur): - nonlocal changed - cur = cur if isinstance(cur, dict) else {} - for table_key, wanted_fields in wanted_by_table.items(): - table = cur.get(table_key) - if table is None: - continue - wanted = {str(f.get("key")): dict(f) for f in wanted_fields} - drops = set(drop_by_table.get(table_key) or ()) - fields, seen = [], set() - drops = _guarded_drops(table, drops) - for stored in table.get("fields") or []: - key = str(stored.get("key") or "") - if key in drops: - changed = True - continue - desired = wanted.get(key) - if desired is None: - fields.append(stored) - continue - # ⭐⭐ 2026-08-09 (owner: *"everything is custom and changeable always"*) — A - # COLUMN A HUMAN HAS TAKEN OVER IS LEFT ALONE. Every key below is overwritten - # from the shipped contract, `rollup` included, so without this the newly - # unlocked "edit a preset rollup" would save, render, recompute and then revert - # at the next enrichment run — an edit that looks like it worked and undoes - # itself hours later. `user_tables.user_edited` is the ONE reader of the stamp. - if _ut().user_edited(stored): - fields.append(stored) - seen.add(key) - continue - repaired = dict(stored) - for dkey, value in desired.items(): - if dkey != "automation": - repaired[dkey] = value - automation = dict(stored.get("automation") or {}) - automation.update(desired.get("automation") or {}) - repaired["automation"] = automation - # These bags define behaviour, not decoration. If the desired field does not use - # one, a stale bag from an earlier type must not remain attached to it. - for bag in ("link", "rollup", "profile", "pinned"): - if bag not in desired: - repaired.pop(bag, None) - if desired.get("type") not in ("select", "multiselect"): - repaired.pop("options", None) - converter = _IG_RETYPE_CONVERTERS.get((table_key, key)) - if (converter and stored.get("type") in ("text", "", None) - and repaired.get("type") != stored.get("type")): - for row in (table.get("rows") or {}).values(): - if str(row.get(key) or "").strip(): - converted = converter(row.get(key)) - if converted: - row[key] = converted - changed = True - if repaired != stored: - changed = True - fields.append(repaired) - seen.add(key) - for key, desired in wanted.items(): - if key not in seen and not any(str(f.get("key") or "") == key for f in fields): - fields.append(desired) - changed = True - if drops: - for row in (table.get("rows") or {}).values(): - for key in drops: - if key in row: - row.pop(key, None) - changed = True - table["fields"] = fields - return cur - - # Avoid a commit when the declarations are already exact. - snapshot = ut_all(rt) - needs = False - for table_key, wanted_fields in wanted_by_table.items(): - if table_key not in snapshot: - continue - table = snapshot.get(table_key) or {} - by_key = {str(f.get("key") or ""): f for f in table.get("fields") or []} - # ⚠ THE SAME GUARD, OR THE MIGRATION STOPS BEING IDEMPOTENT. This precheck exists to skip - # the commit when nothing would change; a spared `tracked` column left in the drop set - # answers "yes, work to do" on every single call, forever, and the schema pass would - # rewrite the table on every authenticated read without ever changing a byte. - drops = _guarded_drops(table, drop_by_table.get(table_key) or ()) - if drops & set(by_key): - needs = True - break - for desired in wanted_fields: - stored = by_key.get(str(desired.get("key") or "")) - mismatch = stored is None - if stored is not None: - for key, value in desired.items(): - if key == "automation": - if any((stored.get("automation") or {}).get(k) != v - for k, v in value.items()): - mismatch = True - break - elif stored.get(key) != value: - mismatch = True - break - if not mismatch: - stale_bags = {"link", "rollup", "profile", "pinned"} - set(desired) - mismatch = any(k in stored for k in stale_bags) - if (not mismatch and desired.get("type") not in ("select", "multiselect") - and "options" in stored): - mismatch = True - if mismatch: - needs = True - break - if needs: - break - if needs: - rt.update(UT_STORE_KEY, _up, flush="sync") - return changed - - -def ensure_ig_graph(rt, username="automation", flow_tag="", profile_table="", profile_field=""): - """Ensure the ONE per-tenant Instagram relational graph and return its canonical keys. - - Every enrichment path calls this function. Fixed table keys prevent a flow aimed at a new - Profile database from spawning `IG posts 2`; deterministic reciprocal link fields connect - each Profile database to the same Posts, Comments, profile-history, and post-history stores. - """ - profile = ut_get(rt, profile_table) if profile_table else None - bound = str(profile_field or "").strip() - if profile is not None and not bound: - bound = next((str(f.get("key") or "") for f in profile.get("fields") or [] - if isinstance(f.get("profile"), dict)), "") - profile_label = str((profile or {}).get("label") or profile_table) - backlink = [_profile_backlink_field(profile_table, profile_label, bound)] \ - if profile_table and profile is not None and bound else [] - - # ⭐ WAVE 32 · T41 — DERIVED from `IG_TABLE_FIELDS`/`IG_TABLE_LABELS`, which used to be this - # literal. The backlinks stay HERE because they are per-tenant, per-profile-database facts; the - # schema is a module constant. Splitting it that way is what let A's delivery sweep ask this - # module for Instagram's child set instead of hand-typing a fifth copy of these four names. - graph = {key: (IG_TABLE_LABELS[key], [*fields, *backlink]) - for key, fields in IG_TABLE_FIELDS.items()} - keys = {} - for key, (label, fields) in graph.items(): - keys[key] = ut_ensure(rt, label, fields, username, key=key, flow_tag=flow_tag, - record_mode=AUTOMATION_RECORD_MODE, lock_fields=True) - - if profile_table and profile is not None and bound: - profile_fields = _profile_schema_for(bound) - ut_ensure(rt, profile_label or profile_table, profile_fields, username, - key=profile_table, flow_tag=flow_tag, lock_fields=True) - # Reconcile the WHOLE tenant graph, not just this run's target. That is what keeps an older - # Profile database and IG candidates on the same universal schema while all of them point to - # the same four children. Existing flow provenance is merged, never replaced. - wanted, drops = _ig_schema_contract(rt) - _reconcile_ig_graph_fields(rt, wanted, drops) - return keys - - -# ── ⭐ WAVE 26 · THE MIGRATION (owner rulings R3 + R4/R5, contracts C1-a and C3) ─────────────── -# -# Two changes land on data that already exists, and neither is optional once the field defs move: -# -# R3 the preset columns take honest types, so the CELLS have to match them — an ISO stamp in a -# `date` column and a "20872" string in an `int` column are the split-schema the old -# "types are text on purpose" note correctly feared. Re-typing without converting is worse -# than not re-typing. -# R4 the candidate identity becomes `(platform, handle)`, so rows that exist today as -# `(handle, created_by)` DUPLICATES must be merged rather than left to collide. -# -# ⛔ IDEMPOTENCY IS THE WHOLE DESIGN, and the key is the STORED FIELD TYPE — never a flag, never a -# marker column. Run twice, an `avg_engagement` of 0.0074 would go 0.74 then 74, and nothing would -# look wrong until somebody read a 74% engagement rate off a creator with 3,000 followers. So a -# column is converted only while its STORED definition still says `text`; the moment it says `pct` -# the work is done and re-running is a no-op. That makes the migration safe to call on every boot, -# which is what it is for. -# -# ⚠ THE TABLE LIST IS DERIVED, NOT NAMED — D-71's lesson, one wave old and already paid for twice. -# `observed_categories` named its two source tables and went silently empty the day wave 25 let a -# user point an automation somewhere else. Naming tables here would leave exactly those user-named -# databases on the old schema, which is the same bug wearing a migration's clothes. - -#: A table is an Instagram profile table if it declares `handle` plus a real slice of the preset -#: vocabulary. Deliberately structural: it finds `ut_beauty_influencer_leads` without being told. -MIGRATE_MIN_PRESET_FIELDS = 3 -#: key -> the converter its new type needs. `checkbox` needs none (the writers already emit '1'/''). -_MIGRATE_CONVERT = { - "first_found": _day, "last_found": _day, "enriched_at": _day, - "avg_engagement": _pct100, -} - - -#: ⭐⭐ WAVE 30 · T08 — THE COLUMNS THAT CAN ONLY BE TIKTOK'S, DERIVED rather than typed out, so -#: that adding a column to either declaration keeps this set correct instead of quietly emptying -#: it. Eight today: `tt_id`, `like_engagement`, `comment_engagement`, `likes_received`, `region`, -#: `predicted_lang`, `account_created_at`, `source`. -TT_ONLY_PROFILE_KEYS = (frozenset(f["key"] for f in TT_PROFILE_FIELDS) - - frozenset(PRESET_PROFILE_KEYS) - - frozenset(f["key"] for f in CANDIDATE_FIELDS)) - - -def _is_tt_profile_table(tbl): - """⭐⭐ WAVE 30 · T08 — is this database a TIKTOK profile table? - - ⛔ THE DEFECT THIS EXISTS TO CLOSE WAS LIVE AND SHIPPED, and it converted a customer's TikTok - database into an Instagram one on the SECOND write to it. MEASURED: a `ut_tt_profile` spawned - by W30-T06 comes out correct — `handle` carrying `profile: {source: "tiktok"}` — and one more - `ut_ensure` on that table (a re-save, the discovery runner's own ensure, the next run) leaves - it carrying `profile: {source: "instagram"}` and the description *"Instagram username without - the @ symbol"*. `profile_field_key(..., source="tiktok")` then answers `""` forever, so every - TikTok enrich step on that database reports UNBOUND, and every row gets stamped - `platform: Instagram` — which is half of W26/R4's `(platform, handle)` identity, so two - different people's accounts merge under one key with nothing red anywhere. - ⚠ It hid because the migration SKIPS a table that does not exist yet, so the spawn itself is - always clean and only the second write is not. A gate that creates and asserts once cannot see - it; the assertion has to survive a second `ut_ensure`. - - ⭐ TWO LEGS, and the second is the one that still works after the first has been eaten: - 1. the table SAYS SO — a column declaring `profile: {source: "tiktok"}` is the database's own - statement of which network it is about, and it is the same declaration `profile_field_key` - reads, so there is one answer to "whose profile table is this"; - 2. it declares a column only TikTok has. Needed because leg 1 is exactly what the defect - destroys: on a table already corrupted in production, the flag now says Instagram, and a - detector resting on it alone would agree with the corruption and keep re-applying it. - """ - fields = (tbl or {}).get("fields") or [] - for f in fields: - p = f.get("profile") - if isinstance(p, dict) and str(p.get("source") or "") == PROFILE_SOURCE_TT: - return True - return bool({f.get("key") for f in fields} & TT_ONLY_PROFILE_KEYS) - - -#: ⭐⭐ WAVE 32 · T40 — THE MIRROR OF `TT_ONLY_PROFILE_KEYS`, and it is the subject of owner item 1: -#: the profile columns that can only ever be INSTAGRAM's. 26 today. Derived by the same subtraction -#: in the other direction, so a column moved between the two declarations changes both sets at once -#: rather than leaving one of them quietly asserting a stale fence. -#: ⚠ IT IS DERIVED FROM `CANDIDATE_FIELDS`, NOT `PRESET_PROFILE_FIELDS` — discovery's bookkeeping -#: columns (`found_count`, `first_found`, `last_found`, `created_by`) are declared on BOTH platforms, -#: so subtracting the preset list alone would report four keys as Instagram-only that TikTok's own -#: table has carried since wave 29. -IG_ONLY_PROFILE_KEYS = (frozenset(f["key"] for f in CANDIDATE_FIELDS) - - frozenset(f["key"] for f in TT_PROFILE_FIELDS)) - -#: ⭐⭐ WAVE 32 · T42 — the preset columns NOBODY TYPES: every `link` and `rollup` in the Instagram -#: preset set. Derived from the declaration, so adding a rollup to the contract widens this for -#: free. `_carries_ig_presets` uses it as the leg that survives when the `automation.preset` stamp -#: is absent — a hand-made creator list has `handle` and `followers`; it does not have `posts_link`. -PRESET_MACHINE_ONLY_KEYS = frozenset( - f["key"] for f in PRESET_PROFILE_FIELDS if f.get("type") in ("link", "rollup")) - - -def platform_schema(platform): - """⭐⭐ WAVE 32 · T41 — ONE ASKABLE DECLARATION OF WHAT A PLATFORM'S DATABASES ARE. - - Returns, for `PLATFORM_INSTAGRAM` or `PLATFORM_TIKTOK`: - - {"platform", "profile_table", "profile_label", "profile_fields", "preset_keys", - "only_keys", "locked_tables", "children": {key: {"label", "fields", "record_mode"}}} - - ⛔ **WHY IT IS A FUNCTION AND NOT A DICT LITERAL — the same import-order trap `discovery_facts` - records twenty lines from its own declaration.** `CANDIDATE_FIELDS` and `DISCOVER_TABLE` are - declared HUNDREDS of lines below `TT_TABLE_FIELDS`, so any module-level dict spanning both - platforms NameErrors at import. A function body resolves at call time and does not care. That is - not a style preference here; it is the reason two earlier attempts at a platform registry in this - file ended up as five inline copies instead. - - ⛔⛔ **AND IT IMPORTS NOTHING FROM `core`, WHICH IS THE ACTUAL CONTRACT WITH `main.py`.** This - module is deliberately dependency-light on the API's boot path (`MAX_UT_ROWS`, `MACHINE_OWNERS` - and `LOCKED_CHILD_TABLES` all say so in their own notes), and the REGISTRAR — the thing that - turns these names into `core.user_tables` state — lives in the composition root. So this returns - plain lists and dicts: a caller can `register_locked_records(...)`, `ut_ensure(...)` or diff it - against a live tenant without `core` ever appearing on the engine's import path. `verify_ - automation` asserts that, by importing this module in a CLEAN interpreter and checking - `core.user_tables` is absent from `sys.modules` afterwards — [[artifact-with-no-importer]] in - reverse: a declaration whose registrar cannot reach it is the same defect as a registrar with - nothing to register. - - ⚠ FIELD DICTS ARE COPIED ONE LEVEL. Every existing call site passes the module lists straight - into `ut_ensure`, which is safe only because nobody has yet appended to one; a boot sweep looping - over both platforms is exactly the caller that would. The copy costs a few hundred dict - constructions once per boot and removes the whole question. - - ⚠ `profile_fields` is the platform's PRESET set as spawned — Instagram's is `CANDIDATE_FIELDS` - (the preset set PLUS discovery's bookkeeping), not `PRESET_PROFILE_FIELDS`, because that is what - `discovery_facts` actually hands the spawn. Two answers to "Instagram's field set" is the drift - this function exists to end, so it gives the one the product uses. - """ - name = str(platform or "") - if name == PLATFORM_TIKTOK: - children = {k: {"label": TT_TABLE_LABELS[k], "fields": [dict(f) for f in TT_TABLE_FIELDS[k]], - "record_mode": tt_record_mode(k)} - for k in sorted(TT_TABLE_FIELDS) if k != TT_PROFILE_TABLE} - return { - "platform": PLATFORM_TIKTOK, - "profile_table": TT_PROFILE_TABLE, - "profile_label": TT_TABLE_LABELS[TT_PROFILE_TABLE], - "profile_fields": [dict(f) for f in TT_PROFILE_FIELDS], - "preset_keys": tuple(f["key"] for f in TT_PROFILE_FIELDS), - "only_keys": TT_ONLY_PROFILE_KEYS, - "locked_tables": TT_LOCKED_TABLES, - "children": children, - } - if name == PLATFORM_INSTAGRAM: - children = {k: {"label": IG_TABLE_LABELS[k], "fields": [dict(f) for f in fields], - # ⚠ NOT `tt_record_mode`'s twin by accident: `ensure_ig_graph` passes - # `AUTOMATION_RECORD_MODE` for all four unconditionally, and - # `IG_LOCKED_TABLES` is exactly those four. Derived from the SET so that - # unlocking one there unlocks it here, rather than from the constant. - "record_mode": (AUTOMATION_RECORD_MODE if k in IG_LOCKED_TABLES else "")} - for k, fields in sorted(IG_TABLE_FIELDS.items())} - return { - "platform": PLATFORM_INSTAGRAM, - "profile_table": DISCOVER_TABLE, - "profile_label": DISCOVER_LABEL, - "profile_fields": [dict(f) for f in CANDIDATE_FIELDS], - "preset_keys": tuple(f["key"] for f in CANDIDATE_FIELDS), - "only_keys": IG_ONLY_PROFILE_KEYS, - "locked_tables": IG_LOCKED_TABLES, - "children": children, - } - raise ValueError(f"no schema is declared for platform {platform!r}") - - -def definition_platforms(defn): - """Which networks ONE automation uses — `{"Instagram"}`, `{"TikTok"}`, both, or empty. - - ⭐ WAVE 32 · T42. Three ways a definition names a network, and all three count, because the - owner's rule is about what a database is USED FOR rather than about how the automation was - built: its discovery KIND (a corpus search is a network by construction), its enrich ACTIONS - (a `plain` flow that enriches is using that network), and the retired `field_instagram` kind, - which is uncreatable but alive on stored definitions (D-65) and still runs Instagram. - """ - defn = defn or {} - kind = str(defn.get("kind") or "") - out = set() - if kind in DISCOVERY_KINDS: - out.add(discovery_facts(kind)[0]) - if kind == "field_instagram": - out.add(PLATFORM_INSTAGRAM) - actions = (defn.get("flow") or {}).get("actions") or [] - if _actions_of_kind(actions, "enrich_instagram"): - out.add(PLATFORM_INSTAGRAM) - if _actions_of_kind(actions, "enrich_tiktok"): - out.add(PLATFORM_TIKTOK) - return out - - -def automation_platforms_by_table(rt, definitions=None): - """⭐⭐ WAVE 32 · T42 (owner item 1, third clause) — WHICH NETWORKS EACH DATABASE IS USED FOR. - - `{table_key: {"Instagram", "TikTok"}}`, over every stored automation. The owner's words are the - whole specification: *"if a user decide to use one database for both Tiktok and Instagram - scraping, only then can the pre-set Fields can exist in the same database."* — so the question a - schema pass has to be able to ask is *"which networks target THIS database"*, and until now - nothing in this module could answer it. - - ⛔ ASKABLE AND `core`-FREE, for `platform_schema`'s reason one function up: it reads only the - automations bucket through `all_definitions`, returns plain sets, and the T41 subprocess probe - covers it — a second askable declaration that quietly needed `core` would make the import-purity - guarantee mean "true for the things we remembered". - - ⚠ `definitions` is a LENT list, same contract as `ut_ensure`'s `tables=`: a caller already - holding the bucket must not pay for a second deep copy of it. - """ - # ⚠ `all_definitions` answers a DICT keyed by id, not a list — and iterating it directly hands - # every consumer a STRING that looks like a definition until the first `.get`. Normalised here - # so a caller lending its own list gets the same treatment. - defs = all_definitions(rt) if definitions is None else definitions - defs = list(defs.values()) if isinstance(defs, dict) else list(defs or []) - out = {} - for defn in defs: - if not isinstance(defn, dict): - continue - table = str(_flow_table(defn) or (defn.get("config") or {}).get("targetTable") or "") - if not table: - continue - found = definition_platforms(defn) - if found: - out.setdefault(table, set()).update(found) - return out - - -def platform_schemas(): - """Both declarations, in `PLATFORMS` order — what a boot sweep loops over. - - ⚠ `PLATFORM_FACEBOOK` is in `PLATFORMS` (it is a value the `platform` COLUMN may hold) and has - no schema, so this filters rather than raising: a caller sweeping every declared platform must - not be broken by a vocabulary entry that names no databases. `platform_schema` still raises for - it, which is the right answer to somebody ASKING for a schema that does not exist. - """ - out = [] - for name in PLATFORMS: - try: - out.append(platform_schema(name)) - except ValueError: - continue - return out - - -def _ig_profile_tables(rt, tables=None): - """Every `ut_*` table in this tenant that looks like an Instagram profile table. - - ``tables`` is the optional lent snapshot (W29-T01): read-only walk, so a caller that already - holds the bucket must not pay for a second deep copy of it. - - ⛔ WAVE 30 · T08 — AND THE TEST IS NOW EXCLUSIVE, because "looks like" was network-blind and a - TikTok profile table looks EXACTLY like one: 18 of `ut_tt_profile`'s 30 columns are Instagram - preset keys, and the bar here is `handle` plus three. Every caller of this function then treats - the match as an Instagram table — `migrate_ig_tables` rewrites its declarations to Instagram's - contract, and `discover_default_table` offers it as the default target for an Instagram search. - """ - out = [] - for key, tbl in sorted(((ut_all(rt) if tables is None else tables) or {}).items()): - keys = {f.get("key") for f in (tbl or {}).get("fields") or []} - if "handle" not in keys or len(keys & set(PRESET_PROFILE_KEYS)) < MIGRATE_MIN_PRESET_FIELDS: - continue - if _is_tt_profile_table(tbl): - continue - out.append(key) - return out - - -def _ig_contract_tables(rt, tables=None): - """⭐⭐ WAVE 32 · T42 — the tables Instagram's preset CONTRACT may be applied to. - - ⛔⛔ THIS IS A DIFFERENT QUESTION FROM `_ig_profile_tables` AND CONFLATING THEM COST FIVE RED - CHECKS. That function answers *"does this database LOOK like an Instagram profile table?"* and - two callers need exactly that: `discover_default_table`, which elects an existing table so a new - automation does not mint a second empty one — and it necessarily runs BEFORE any automation - targets that table, so a targeted-only test makes it elect nothing forever — and the detector's - own negative control. THIS function answers *"may we write Instagram's 48 columns into it?"*, - which is the owner's rule and a strictly narrower set. One normalizer answering two questions is - how a fix in one becomes a silent regression in the other ([[one-question-two-normalizers]]). - - A table qualifies when it looks like one AND any of: - - it is `ut_ig_profile`, Instagram's canonical database by declaration; - - it ALREADY carries the contract (`_carries_ig_presets`) — narrowing must never orphan a - table that has the columns, or its links and rollups silently stop being repaired; - - an INSTAGRAM automation targets it, which is the owner's rule stated in code. - - ⚠ The automations bucket is read LAZILY, on the first candidate that needs the question asked — - a tenant whose tables all already carry the contract pays nothing, on a path this wave is - otherwise trying to make faster. - """ - targets = None - out = [] - for key in _ig_profile_tables(rt, tables=tables): - tbl = (ut_all(rt) if tables is None else tables).get(key) or {} - if key != DISCOVER_TABLE and not _carries_ig_presets(tbl): - if targets is None: - targets = automation_platforms_by_table(rt) - if PLATFORM_INSTAGRAM not in targets.get(key, set()): - continue - out.append(key) - return out - - -def _preset_owned(field): - """Is this column MACHINE-authored, i.e. may `retract_foreign_presets` delete it? - - ⛔ MODULE-LEVEL, NOT A CLOSURE, AND THAT IS WHY IT MOVED. It was a nested `_machine` and the - negative control written against it came out BLIND: nothing outside the function could break - the one guard standing between a boot-time repair and a customer's own column, so the control - ended up patching the key SETS instead and proved something else. A guard a control cannot - reach is a guard nobody has tested [[gate-negative-control]]. - - Two legs, the second for tables that predate the stamp: the `automation.preset` mark that - `ut_ensure(lock_fields=True)` and `_locked_ig_field` both write, or a key that is a preset - LINK/ROLLUP — nobody types `posts_link` by hand. - """ - automation = (field or {}).get("automation") - if isinstance(automation, dict) and automation.get("preset") is True: - return True - return str((field or {}).get("key") or "") in PRESET_MACHINE_ONLY_KEYS - - -def _filled_count(rows, key): - """How many rows carry a non-blank value in `key` — the number that decides REPORT vs DELETE. - - ⛔ ALSO MODULE-LEVEL FOR ITS CONTROL'S SAKE. This one number is the whole of W30/R6's second - sentence inside `retract_foreign_presets`: a foreign column with cells in it is reported, never - deleted. A control that cannot make this lie cannot prove the report is doing anything. - """ - return sum(1 for r in (rows or {}).values() - if isinstance(r, dict) and str(r.get(key) or "").strip()) - - -def retract_foreign_presets(rt, log=print): - """⭐⭐ WAVE 32 · T46 / DEBT D-152 — REMOVE THE PRESET COLUMNS OF A NETWORK A DATABASE IS NOT - USED FOR. The RETRACTION half of `_ig_contract_tables`' recruitment rule, and A's boot sweep - (`W32-T07`) calls it once per tenant. - - ⛔ WHY IT HAS TO EXIST AT ALL, and this is the wave's own thesis one layer down. W30-T08 fixed - the DETECTOR, so the corruption cannot recur — and nothing undoes it. MEASURED on a fixture: a - `ut_tt_profile` corrupted before that fix keeps ALL 26 Instagram-only columns and its - `profile: {source: "instagram"}` flag through `ut_ensure`, `migrate_ig_tables` AND the discovery - spawn. Three write paths, zero repairs. *"The corruption cannot recur"* and *"the corruption is - gone"* are different claims [[a-migration-that-runs-on-the-next-write]]. - - Returns `{"tables", "columns", "cells", "flags", "kept"}`. **`kept` is the important one** — - see guarantee 4. - - THE FOUR GUARANTEES, in the order they matter: - 1. **Only machine-authored columns go.** A field must be stamped `automation.preset: True` - (or be a link/rollup this contract owns) to be eligible. A column a PERSON typed is never - touched, whatever it is named — the D-152 row calls this *"a VALUE BACKFILL over live - customer rows"*, and that is the line it must not cross. - 2. **Only the EXCLUSIVE sets.** `IG_ONLY_PROFILE_KEYS` (26) and `TT_ONLY_PROFILE_KEYS` (8), - both derived. A key both platforms declare — `platform`, `handle`, `followers`, the four - discovery bookkeeping columns — is shared vocabulary and is never a foreign column. - 3. **Idempotent.** A second call finds nothing and writes nothing, so it is safe on every - boot, which is what makes it callable from `main.py` rather than being a script somebody - has to remember to run (the whole reason D-201 is still open). - 4. ⛔ **A FOREIGN COLUMN THAT HOLDS DATA IS REPORTED, NOT DELETED.** W30/R6's second sentence: - a limit that cannot be removed is reported with its cause. Deleting a customer's populated - cells to make a grid look clean is not a repair, and this is precisely the *"deserves its - own supervision"* clause of D-152. `kept` names every such column with its row count, so - the caller can print it and a person can decide. - - ⚠ THE FLAG IS REPAIRED TOO, and it is the half D-152 names explicitly: a TikTok profile table - whose `handle` was re-declared `profile: {source: "instagram"}` answers `""` to - `profile_field_key(..., source="tiktok")` forever, so every TikTok enrich on it reports UNBOUND. - Repaired only where `_is_tt_profile_table` recognises the table by its OWN TikTok-only columns, - i.e. by the leg the corruption cannot have eaten. - """ - stats = {"tables": 0, "columns": 0, "cells": 0, "flags": 0, "kept": []} - targets = automation_platforms_by_table(rt) - exclusive = {PLATFORM_INSTAGRAM: IG_ONLY_PROFILE_KEYS, - PLATFORM_TIKTOK: TT_ONLY_PROFILE_KEYS} - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - for key, tbl in sorted(cur.items()): - if not isinstance(tbl, dict): - continue - fields = tbl.get("fields") or [] - keys = {str(f.get("key") or "") for f in fields} - # Only PROFILE databases are in scope: `handle` plus a real slice of either preset - # vocabulary. A posts or comments table shares no profile columns and is never a - # candidate; a database with nothing in common with either is not one either. - if "handle" not in keys: - continue - tt = _is_tt_profile_table(tbl) - used = set(targets.get(key) or ()) - if not used: - # Nothing targets it. Its own declaration is then the only statement of what it is - # for — and a table that says TikTok is not an Instagram table, whatever columns a - # past migration wrote into it. - used = {PLATFORM_TIKTOK} if tt else {PLATFORM_INSTAGRAM} - foreign = set() - for platform, own in exclusive.items(): - if platform not in used: - foreign |= set(own) - if not foreign: - continue - rows = tbl.get("rows") or {} - drop, touched = [], False - for f in fields: - fkey = str(f.get("key") or "") - if fkey not in foreign or not _preset_owned(f): - continue - filled = _filled_count(rows, fkey) - if filled: - # Guarantee 4 — REPORTED, never deleted. - stats["kept"].append({"table": key, "column": fkey, "rows": filled}) - continue - drop.append(fkey) - if drop: - tbl["fields"] = [f for f in fields if str(f.get("key") or "") not in drop] - for r in rows.values(): - if not isinstance(r, dict): - continue - for fkey in drop: - if fkey in r: - r.pop(fkey, None) - stats["cells"] += 1 - stats["columns"] += len(drop) - touched = True - # ⚠ THE FLAG, and only on a table TikTok's own columns still identify. - if tt: - for f in tbl.get("fields") or []: - p = f.get("profile") - if isinstance(p, dict) and str(p.get("source") or "") == PROFILE_SOURCE_IG: - f["profile"] = {**p, "source": PROFILE_SOURCE_TT} - stats["flags"] += 1 - touched = True - if touched: - stats["tables"] += 1 - return cur - - # ⚠ The precheck is the WHOLE mutation, run against a snapshot, so the common case (nothing to - # do) spends no commit — the same posture `_reconcile_ig_graph_fields` takes, and the reason - # this is safe to call on every boot. - probe = copy.deepcopy(ut_all(rt)) - _up(probe) - if not (stats["columns"] or stats["flags"]): - if stats["kept"]: - log(f"[aios-auto] retract: {len(stats['kept'])} foreign column(s) KEPT because they " - f"hold data. " + ", ".join(f"{k['table']}.{k['column']} ({k['rows']} rows)" - for k in stats["kept"])) - return stats - stats = {"tables": 0, "columns": 0, "cells": 0, "flags": 0, "kept": []} - rt.update(UT_STORE_KEY, _up, flush="sync") - log(f"[aios-auto] retract: {stats['columns']} foreign column(s), {stats['cells']} cell(s) and " - f"{stats['flags']} mis-declared flag(s) across {stats['tables']} database(s)") - if stats["kept"]: - log(f"[aios-auto] retract: {len(stats['kept'])} foreign column(s) KEPT because they hold " - f"data. " + ", ".join(f"{k['table']}.{k['column']} ({k['rows']} rows)" - for k in stats["kept"])) - return stats - - -def _carries_ig_presets(tbl): - """Has this table ALREADY been given Instagram's preset contract? - - ⭐⭐ WAVE 32 · T42 — THE RECRUIT/REPAIR LINE, AND IT RESTS ON A STAMP THE CODE ALREADY WRITES. - `ut_ensure(lock_fields=True)` and `_locked_ig_field` both set `automation: {preset: True}` on - every preset column, and `migrate_ig_tables` BACKFILLS it on a table that predates the stamp - (MEASURED: an unstamped 44-column fixture comes out 44/44 stamped after one pass). So "already - carries the contract" is a declaration to read, not a threshold to invent — and inventing one - was the alternative, because `MIGRATE_MIN_PRESET_FIELDS` is 3 and cannot tell a recruited table - from a hand-made creator list that happens to have `handle`, `followers` and `bio`. - - ⛔ WHY THE LINE EXISTS AT ALL (owner item 1's third clause, MEASURED): a hand-made 5-column - database that NO automation targets went to **45 columns** the moment an Instagram automation - aimed at a DIFFERENT database was SAVED — because `_ig_schema_contract` walks every - structurally-matching table in the tenant and hands each one the full 48-column contract. The - owner's rule is that the preset fields may live in a database only if it is used for that - scraping; recruitment by resemblance is the opposite of that rule. - - ⚠ NARROWING NEVER DROPS A COLUMN. `_reconcile_ig_graph_fields` deletes only the fixed retired-key - `drops` set, so a de-recruited table keeps every field and cell it has — it stops being ADDED to - and REPAIRED, which is the whole of the change. Asserted on a fixture rather than reasoned. - ⚠ AND THE COST IS PAID ONLY WHEN IT HAS TO BE: `_ig_profile_tables` reads the automations bucket - lazily, on the first candidate that is neither stamped nor the canonical `ut_ig_profile` — so a - tenant whose tables are all genuinely Instagram's pays nothing on a path this wave is otherwise - trying to make faster. - - ⛔⛔ TWO LEGS, AND THE SECOND IS THE ONE THAT KEEPS THIS FROM BREAKING A LIVE TENANT. The stamp - is written by the very passes this predicate now gates, so a table recruited BEFORE the stamp - existed and targeted by no surviving automation would be de-recruited and never stamped — - silently losing its rollup/link repair, which is the slowest and worst failure available here - (MEASURED on a stripped fixture: 44 columns, 0 stamps, 0 repaired). So a table ALSO counts as - already-carrying when it declares one of the preset set's LINK or ROLLUP columns. Those are - machine-authored by construction — nobody types `posts_link` or `avg_views_12` into a hand-made - creator list — so the second leg is derived from the contract itself, not a threshold somebody - picked. [[gate-and-nc-must-not-share-a-binding]]'s cousin: a discriminator written by the thing - it discriminates needs an independent second leg, exactly as `_is_tt_profile_table` needed one. - """ - fields = (tbl or {}).get("fields") or [] - for f in fields: - if str(f.get("key") or "") not in PRESET_PROFILE_KEYS: - continue - automation = f.get("automation") - if isinstance(automation, dict) and automation.get("preset") is True: - return True - return bool({str(f.get("key") or "") for f in fields} & PRESET_MACHINE_ONLY_KEYS) - - -def discover_default_table(rt, tables=None): - """Which database should a discovery automation write to when nobody has said? (2026-08-10) - - Owner: *"the damn database is supposed to be dynamic for whatever instagram profile is there. - We only need ONE IG profile so it's not confusing."* - - ⛔ THE COMPLAINT WAS ABOUT A SECOND EMPTY DATABASE, NOT A HARDCODED KEY. `ut_beauty_influencer_ - leads` appears NOWHERE in this product as a literal — it is a slug `ut_ensure` minted from the - label the tenant typed, and `_ig_profile_tables` finds it structurally (that function's own - note says so). What IS hardcoded is `DISCOVER_TABLE`, the FALLBACK target — and because it is - a constant rather than a question, a tenant that already had an Instagram profile database got - a SECOND, empty one the first time a discovery automation was saved without a target. - MEASURED on nurilab: `ut_ig_candidates` (0 rows, 44 preset columns) sitting beside - `ut_beauty_influencer_leads` (105 rows), both matching the profile predicate, one of them - pure confusion. - - So the default becomes a question asked of the tenant: - - exactly one profile database (excluding the fallback) -> that one - several, exactly one of which holds rows -> the one in use - several in use, or none at all -> "" (the caller keeps DISCOVER_TABLE) - - ⚠ RETURNS "" RATHER THAN GUESSING. Two populated profile databases is a real ambiguity and - picking one silently would write a paid discovery run into a database the user did not name — - worse than the empty table this exists to prevent. "" means "no opinion", and the caller's - existing fallback stands, which is exactly today's behaviour for that case. - ⚠ THE FALLBACK IS EXCLUDED FROM ITS OWN ELECTION. `ut_ig_candidates` carries the full preset - schema, so it matches `_ig_profile_tables` — without this line a tenant that already has the - empty table would keep re-electing it and nothing would ever change. - - ⭐ WAVE 29 (W29-T01) — ONE READ, NOT TWO. This function read the whole `user_tables` bucket - here and `_ig_profile_tables` read it AGAIN one line below: two 35.8 MB-ceiling deep copies to - answer one question, on a route the automation surface calls on every open. The election is - now computed from a single snapshot, which is also the only way it can be internally - consistent — the two reads could disagree with each other under a concurrent write. - ``tables`` lets `GET /automations` lend the copy it already holds; the answer is still derived - fresh on every call, so there is no memo to invalidate when a database is created or deleted. - """ - try: - tables = (ut_all(rt) if tables is None else tables) or {} - found = [k for k in _ig_profile_tables(rt, tables=tables) if k != DISCOVER_TABLE] - except Exception: # noqa: BLE001 - return "" - if len(found) == 1: - return found[0] - if len(found) > 1: - used = [k for k in found if (tables.get(k) or {}).get("rows")] - if len(used) == 1: - return used[0] - return "" - - -def _apply_discover_default(rt, raw): - """Fill in a discovery automation's target BEFORE the pure validator invents one. - - ⛔ IT HAS TO HAPPEN HERE, AND THE REASON IS THE WHOLE DESIGN. `clean_config` is the thing that - turns a missing target into `DISCOVER_TABLE` — and it takes no `rt`, by design (it is a pure - validator with a two-argument contract every gate and route depends on). Resolving only at the - RUN sites would be cosmetic: the literal is written into the STORED config at save time, so - `cfg.get("targetTable")` is truthy forever after and no runtime resolver is ever consulted. - `create`/`patch` are the one pair that both know the tenant and sit above that validator. - ⚠ ONLY WHEN THE CALLER SAID NOTHING. An explicit target — including one a previous save - stored — is the user's choice and is never rewritten. - """ - cfg = raw.get("config") - if not isinstance(cfg, dict) or str(cfg.get("targetTable") or "").strip(): - return raw - default = discover_default_table(rt) - if not default: - return raw - return {**raw, "config": {**cfg, "targetTable": default}} - - -def _vestigial_name_field(fields, rows): - """⭐ 2026-08-07 — the born-blank `Name` column, or None. The owner's *"REPLACED"* half. - - `user_tables.create()` mints EXACTLY ONE column on a hand-made database — - `{key:'name', label:'Name', type:'text', default:True}` — and `ut_ensure` then APPENDS the - automation's columns after it, never reordering. So on every Instagram database somebody - created before pointing an automation at it, column 1 is a `Name` nothing will ever write. - - ⛔ IT IS ONLY VESTIGIAL IF NOBODY EVER USED IT, and the test is deliberately conservative - because this is the one branch in the migration that DESTROYS something. A column carrying any - declaration (an automation binding, a metric, a profile flag) is somebody's work; a column with - a value in any row is somebody's data. Either way it survives as an ordinary column and merely - stops being the locked primary, which the `pinned` half achieves on its own. - - ⚠ `key == 'name'` IS ALREADY DECISIVE and the rest is belt: a user-created column is minted - `custom__` (`buildOverlayField`) and every machine one comes from a `field_def` - list, so nothing but `create()`'s default can hold this key. The label and type are checked - anyway — a renamed or retyped column is a column someone touched on purpose. - """ - f = next((f for f in fields if f.get("key") == "name"), None) - if f is None or f.get("label") != "Name" or f.get("type") != "text": - return None - if any(f.get(k) for k in ("automation", "metric", "profile", "measure", "formula")): - return None - if any(str(r.get("name") or "").strip() for r in rows.values()): - return None - return f - - -def _merge_candidates(rows): - """C3's merge rule, applied to rows now colliding on `(platform, handle)`. - - Earliest `first_found` wins - latest `last_found` wins - `found_count` SUMS - every other cell - takes the first non-blank, preferring the most recently enriched row. - - ⚠ THE SURVIVING ROW KEEPS THE LOWEST ID. Row ids are referenced by saved views, comments and - board cards; minting a new one would orphan all of them, so a merge picks a survivor rather - than creating one. - """ - groups, keepers = {}, {} - for rid in sorted(rows, key=lambda r: (len(str(r)), str(r))): - r = rows[rid] - h = str(r.get("handle") or "").strip() - if not h: - # ⛔⛔ THESE USED TO BE `continue`d, AND THE OUTPUT REPLACES THE TABLE'S ROWS — so a - # row with no handle was SILENTLY DELETED. Owner, 2026-08-07: *"How come when I add a - # manual handle in my 'Beauty influencer leads' database, after an automation run, it - # gets deleted?"* - # - # A row you add by hand is born blank and stays blank until you type into it, so the - # window is not narrow — it is every row, from creation until the cell is committed. - # And before the profile flag shipped (834decd) a typed handle landed in the TYPIST'S - # OVERLAY stratum, leaving the definition row's `handle` empty forever: the row was - # dropped even after it looked filled in on screen. - # - # ⚠ THE FUNCTION'S JOB IS TO MERGE DUPLICATE CANDIDATES, and a row with no handle is - # not a candidate — it is somebody's row. It cannot collide with anything (there is - # nothing to key it on), so it is carried through UNTOUCHED rather than judged. A - # merge pass that deletes what it cannot classify is not a merge, it is a filter. - keepers[rid] = r - continue - groups.setdefault((str(r.get("platform") or PLATFORM_INSTAGRAM).strip(), h), - []).append((rid, r)) - out, merged = dict(keepers), 0 - for _key, members in groups.items(): - if len(members) == 1: - out[members[0][0]] = members[0][1] - continue - merged += len(members) - 1 - # Most-recently-enriched first, so "first non-blank" prefers the freshest measurement. - ordered = sorted(members, key=lambda m: str(m[1].get("enriched_at") or ""), reverse=True) - keep_id = sorted(rid for rid, _r in members)[0] - acc = {} - for _rid, r in ordered: - for k, v in r.items(): - if k not in acc and str(v or "").strip(): - acc[k] = v - firsts = [str(r.get("first_found") or "").strip() for _i, r in members if r.get("first_found")] - lasts = [str(r.get("last_found") or "").strip() for _i, r in members if r.get("last_found")] - if firsts: - acc["first_found"] = min(firsts) - if lasts: - acc["last_found"] = max(lasts) - total = sum(_ig_int(r.get("found_count")) or 0 for _i, r in members) - if total: - acc["found_count"] = str(total) - out[keep_id] = acc - return out, merged - - -#: ⛔ DEBT D-74 + D-81 (wave 27 item 12) — `tracked` WAS DELETED FROM THE VOCABULARY AND NEVER -#: FROM THE TABLES. W26/R6 removed the column from `CANDIDATE_FIELDS` and from ~14 engine sites -#: *"we already have a Field called stage… we don't need another checkbox for this"* — and -#: MEASURED 2026-08-07 on `royal-imports/aios-nurilab-data`, `ut_beauty_influencer_leads` still -#: declared it, filled on 1 of 41 rows. A checkbox on the owner's flagship database that nothing -#: writes, nothing reads, and anybody can still click. -#: -#: ⚠ THE DECISION WAS WHICH, NOT WHETHER (D-74 stated both futures): sweep the ticks and lose the -#: record of who approved what before the wave, or leave them and let the next hand-made `tracked` -#: column silently inherit them. The wave takes the sweep — R6 deleted the concept, so a surviving -#: tick is not a decision anybody can act on, and an inert cell that reappears in the next EXPORT -#: as a column nobody declared is the worse half. -TRACKED_KEY = "tracked" - - -def _retired_tracked_field(fields): - """The retired `tracked` column when it is still the MACHINE's own, else None. - - ⛔ GUARDED THE WAY `_vestigial_name_field` IS, and for the same reason: this is a branch that - DESTROYS something. `tracked` was minted `{type: "checkbox"}` by the discovery preset, so a - column still declaring a checkbox is the one R6 retired. A column somebody has since RETYPED, - or bound a formula/link/rollup/measure to, is their work wearing an old key — it survives as - an ordinary column and merely stops being fed by anything, which was already true. - - ⚠ THE ASYMMETRY WITH THE TWO KEYS BESIDE IT IS DELIBERATE. `posts` and `post_hashtags` are - dropped unguarded: both are machine-derived cells with no history of anyone editing them. - `tracked` is the one a HUMAN ticked, so it is the one that gets the conservative test. - """ - f = next((f for f in fields if str(f.get("key") or "") == TRACKED_KEY), None) - if f is None or f.get("type") not in ("checkbox", "", None): - return None - if any(f.get(k) for k in ("formula", "link", "rollup", "measure", "metric")): - return None - return f - - -def _guarded_drops(table, drops): - """`drops` minus any key this table is allowed to keep — today, exactly `tracked` (D-81). - - ⛔ THE GUARD HAS TO LIVE WHERE THE DROP HAPPENS, and finding that out cost a red check. - `tracked` is deleted from TWO places: `migrate_ig_tables`' profile loop, which asks - `_retired_tracked_field` whether the column is still the machine's own, and the schema - reconciliation, which drops by KEY with no question asked. Guarding only the first was - decorative — the schema pass runs immediately afterwards on the same table and deleted the - retyped column the guard had just spared. Two paths to one deletion with one of them checked - is the shape where a control looks present and enforces nothing - ([[defects-that-mask-each-other]]). - """ - drops = set(drops or ()) - if TRACKED_KEY in drops: - stored = [f for f in (table or {}).get("fields") or [] - if str(f.get("key") or "") == TRACKED_KEY] - if stored and _retired_tracked_field(stored) is None: - drops.discard(TRACKED_KEY) - return drops - - -def _orphan_tracked_rows(fields, rows): - """Row ids carrying a `tracked` CELL that no field declares — D-74's actual subject. - - ⛔ THE COLUMN AND THE CELLS WERE RETIRED SEPARATELY, WHICH IS WHY THIS IS NOT COVERED BY THE - BRANCH ABOVE. R6 deleted the field DEFINITION from `CANDIDATE_FIELDS`; a table whose - declaration was already dropped keeps every `tracked` key in its row dicts, invisible on every - surface that renders from `fields` — and a migration that only looks at `fields` finds nothing - to do and `continue`s past the table. Storage, export and any future re-declaration of the key - all still see them. - """ - if any(str(f.get("key") or "") == TRACKED_KEY for f in fields): - return [] # the column above owns its own cells - return [rid for rid, r in rows.items() if TRACKED_KEY in (r or {})] - - -def backfill_corpus_snapshots(rt, log=print): - """The BACKFILL half: every profile row holding a measurement with no series behind it gets - its corpus observation. Returns how many were written. - - ⛔ ONE BUILDER, TWO CALLERS — `run_discover_instagram` for rows arriving now, this for rows - that arrived before the forward fix existed. A second row-shape here would be two ideas of - what a corpus observation is, and they would disagree on the next promoted column. - - ⚠ THE CONDITION IS "NO SERIES AT ALL", not "no series for this day", and the narrowness is - deliberate. A profile that HAS been enriched already owns real observations; synthesising a - corpus row dated `last_found` for it could out-rank the exact read in a `latest` rollup and - replace a measured number with a corpus one. The gap this closes is a row with NOTHING behind - it, which is the only case where a corpus observation can only improve matters. - - ⚠ IDEMPOTENT BY THE SNAPSHOT KEY (`@`), never by a flag — so this is safe on - every write, which is what `migrate_ig_tables`' placement requires. After one pass the row has - a series, so it stops matching and the scan costs a dict comparison. - """ - snaps = ut_get(rt, IG_SNAPSHOTS_TABLE) - if snaps is None: - return 0 # no series store — see `append_ig_snapshots` on why we never mint one - known = {str((r or {}).get("influencer_key") or "").strip().lower() - for r in (snaps.get("rows") or {}).values()} - known.discard("") - #: A cell counts as a MEASUREMENT if the snapshot schema has somewhere to put it — derived - #: from `SNAPSHOT_FIELDS` rather than named, for `_ig_profile_tables`' own reason (D-71: a - #: named list goes silently empty the day the schema moves). - measured = tuple(fd["key"] for fd in SNAPSHOT_FIELDS - if fd["key"] not in _SNAPSHOT_OWN_KEYS - and fd.get("type") in ("int", "pct")) - incoming = [] - for table_key in _ig_profile_tables(rt): - for row in ((ut_get(rt, table_key) or {}).get("rows") or {}).values(): - handle = str((row or {}).get("handle") or "").strip().lstrip("@").lower() - if not handle or handle in known: - continue - if not any(str((row or {}).get(k) or "").strip() for k in measured): - continue # nothing was ever read about this row — there is no observation to record - built = corpus_snapshot_row(row) - if built: - incoming.append(built) - known.add(handle) # two profile tables holding one handle write ONE observation - return append_ig_snapshots(rt, incoming, log=log) - - -def backfill_post_snapshot_grain(rt, log=print): - """Copy `posted_at`/`type` off the POST row onto every measurement missing them. - - ⛔ STRUCTURAL, NEVER A FLAG: the row matches when the snapshot's cell is blank AND the post's - is not, so a pass that has already run matches nothing and a post that genuinely has no date - is never re-visited on a promise it cannot keep. Same idempotency discipline as the migration - around it. - - ⚠ IT FILLS BLANKS AND NEVER OVERWRITES. A snapshot's `posted_at` is what the vendor said WHEN - THE MEASUREMENT WAS TAKEN; if a later pull disagrees with the post row, the measurement's own - record of the moment is the one to keep — a fact table that lets a dimension rewrite its - history is not a fact table. - """ - psnaps = ut_get(rt, IG_POST_SNAPSHOTS_TABLE) - posts = ut_get(rt, IG_POSTS_TABLE) - if psnaps is None or posts is None: - return 0 - have = {f.get("key") for f in (psnaps.get("fields") or [])} - carried = [k for k in ("posted_at", "type") if k in have] - if not carried: - return 0 # the columns are not declared yet — `ut_ensure` adds them first - by_shortcode = {} - for row in (posts.get("rows") or {}).values(): - code = str((row or {}).get("shortcode") or "").strip() - if code: - by_shortcode[code] = row or {} - changes = {} - for rid, row in (psnaps.get("rows") or {}).items(): - post = by_shortcode.get(str((row or {}).get("shortcode") or "").strip()) - if not post: - continue - for key in carried: - want = str(post.get(key) or "").strip() - if want and not str((row or {}).get(key) or "").strip(): - changes.setdefault(str(rid), {})[key] = _s(want, 200) - if not changes: - return 0 - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - table = cur.get(IG_POST_SNAPSHOTS_TABLE) - if table is not None: - for rid, values in changes.items(): - table.setdefault("rows", {}).setdefault(rid, {}).update(values) - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") - log(f"[ig-migrate] post-measurement grain: {len(changes)} row(s) gained a post date/kind") - return len(changes) - - -#: ⭐⭐ 2026-08-10 (owner: *"retire the old hardcoded method and replace the Instagram database work -#: with correct Rollup and Link fields"*) — THE PROFILE COLUMNS THAT MAY BECOME ROLLUPS. -#: -#: A column qualifies when the SERIES has somewhere to put it: `SNAPSHOT_FIELDS` carries the same -#: key, so `latest` over `profile_snapshots_link` can re-derive it. That is 27 of the 44 preset -#: columns — the other 17 are identity/bookkeeping (`handle`, `platform`, `enriched_at`), already -#: relational (the 12 links and rollups), or have no observation behind them at all -#: (`location_guess`/`location_confidence`, which WE infer rather than read). -#: -#: ⛔⛔ AND "DERIVABLE" IS NOT "SAFE", WHICH IS THE WHOLE REASON THIS IS A PREDICATE AND NOT A LIST. -#: MEASURED on nurilab before any conversion: a bare `latest` would BLANK 221 filled cells — -#: `avg_engagement` and `category` lose 62 each — because the discovery reads that produced those -#: values were never recorded as observations (the hole this same wave closed going forward, in its -#: historical form: 62 profiles hold exactly ONE observation, the enrichment scrape, dated three -#: days AFTER the discovery run whose numbers are on their row). A migration that converts on a -#: NAMED LIST would do that damage on any tenant whose series is thinner than the list's author -#: assumed. -#: -#: ⇒ So the migration converts a column only where it can PROVE, on this tenant's own rows, that -#: the derived value equals the stored one everywhere. See `_convertible_profile_columns`. -#: ⚠ Re-runnable by design: as the series fills in, later passes convert more. A column that is not -#: safe today is not refused forever, it is simply not converted yet. -IG_DERIVABLE_KEYS = tuple( - fd["key"] for fd in PRESET_PROFILE_FIELDS - if fd.get("type") not in ("link", "rollup") - and fd["key"] not in ("platform", "handle", "enriched_at", "first_found", "last_found", - "found_count", "created_by") - and fd["key"] in {s["key"] for s in SNAPSHOT_FIELDS}) - - -def _derived_profile_bag(key): - """The rollup that replaces the mapper's write of ONE profile column. - - ⚠ `where: is_not_empty` IS NOT DECORATION. A bare `latest` returns the newest observation's - value INCLUDING ITS BLANK, so a cheap pull that did not read the field would erase what an - expensive one learned — the exact asymmetry `upsert_rows`' merge exists to prevent on the - scalar side. `where` runs BEFORE the ranking (contract C1), so this reads "the newest - observation that ACTUALLY READ this field", which is what the materialised cell meant. - MEASURED: it rescues 5 of the 12 columns a bare `latest` would damage; the other 7 need the - observation itself, which is why the guard below is a proof and not a hope. - """ - return {"link": "profile_snapshots_link", "field": key, "fn": "latest", - "sortBy": "pulled_at", "sortDir": "desc", - "where": [{"field": key, "op": "is_not_empty"}], "whereConj": "and"} - - -def _derived_profile_columns(table, snapshots): - """Which of `IG_DERIVABLE_KEYS` should be DECLARED as rollups on THIS table. - - = the ones already converted, PLUS the ones whose derived value equals the stored value on - every row. Returns keys. - - ⛔⛔ IT MUST RE-ASSERT THE ONES ALREADY CONVERTED, and forgetting that is a one-line revert with - a several-hour fuse. `_reconcile_ig_graph_fields` overwrites every contract key it owns — - `rollup` included, by design — so a converted column that this function stopped naming would be - re-typed back to `int`/`text` on the next reconcile, its bag dropped, and its value re-written - by the mapper. The edit would look applied and undo itself later, which is exactly - [[preset-unlock-needs-a-custody-stamp]]'s shape. An already-converted column is therefore - included unconditionally rather than re-proved: its cells ARE the derived values, so a proof - would be comparing the fold against itself. - - ⛔ THE PROOF RUNS PER TENANT, PER COLUMN, AT MIGRATION TIME — never against a list somebody - measured on one workspace. `followers` derives exactly on nurilab and could be thin somewhere - else; `category` is damaged on nurilab and might be perfect elsewhere. The only honest - authority is the data in front of the migration. - ⚠ A column with NO stored values anywhere converts too, and that is deliberate rather than an - oversight: there is nothing to lose, and leaving it materialised would mean the schema differs - between two tenants for no reason a reader could discover. - """ - fields = {str(f.get("key")): f for f in (table or {}).get("fields") or []} - rows = (table or {}).get("rows") or {} - by_subject = {} - for snap in ((snapshots or {}).get("rows") or {}).values(): - key = str((snap or {}).get("influencer_key") or "").strip().lower() - if key: - by_subject.setdefault(key, []).append(snap or {}) - # The engine's own ordering, once per subject — newest first, unrankable rows last. - ordered = {} - for subject, observations in by_subject.items(): - keyed = [(o, _sort_key(o.get("pulled_at"), "date")) for o in observations] - rankable = [(o, k) for o, k in keyed if k is not None] - rankable.sort(key=lambda ok: ok[1], reverse=True) - ordered[subject] = [o for o, _k in rankable] + [o for o, k in keyed if k is None] - # ⛔ A rollup needs the LINK it folds. A profile table that has not been through the graph - # reconcile has no `profile_snapshots_link`, and declaring one there would mint a column whose - # link resolves to nothing — a blank cell wearing a configured column's clothes. - if "profile_snapshots_link" not in fields: - return [] - out = [] - for key in IG_DERIVABLE_KEYS: - field = fields.get(key) - if not field: - continue - if field.get("type") == "rollup": - out.append(key) # already converted — see the note above on why - continue - declared = str(field.get("type") or "text") - safe = True - for row in rows.values(): - stored = str((row or {}).get(key) or "").strip() - window = [o for o in ordered.get( - str((row or {}).get("handle") or "").strip().lower(), []) - if str(o.get(key) or "").strip() != ""] # the `where` guard, applied here - derived = str(_rollup_fold("latest", [o.get(key) for o in window], declared) - or "").strip() - if stored != derived: - safe = False - break - if safe: - out.append(key) - return out - - -def _as_derived_field(field): - """One preset scalar declaration → the same column declared as a rollup over the series.""" - key = str(field.get("key") or "") - return {**{k: v for k, v in field.items() if k not in ("agg",)}, - "type": "rollup", "rollup": _derived_profile_bag(key)} - - -def migrate_ig_tables(rt, log=print, only=""): - """Bring every Instagram Profile and canonical child table onto one current contract. - - Returns counted changes rather than a reassuring line. Profile cells are converted before - their type declarations change; then the graph reconciliation adds any missing preset - Links/Rollups, repairs child types, stamps locks, adds all reciprocal backlinks, and deletes - only retired machine fields. - - `only` narrows to a single table, which is how `ut_ensure` calls it: the migration then rides - the WRITE PATH, so a table is brought forward immediately before anything appends to it and no - caller has to remember to run anything. - """ - stats = {"tables": 0, "retyped": 0, "converted": 0, "stamped": 0, "merged": 0, - # ⭐ 2026-08-07 (owner ruling) — the primary-column half. Counted separately from - # `stamped` because they answer different questions: that one is "how many ROWS got a - # platform", these are "how many TABLES changed shape". - "pinned": 0, "flagged": 0, "droppedName": 0, - "droppedRecentPosts": 0, "droppedPostHashtags": 0, - "droppedTracked": 0, "droppedTrackedCells": 0, "schema": 0} - want = {f["key"]: f["type"] for f in CANDIDATE_FIELDS} - # ⛔ `only` NARROWS THE SET, IT DOES NOT BYPASS THE TEST — and skipping that cost six red - # checks the first time. `ut_ensure` calls this with the key it is ABOUT TO CREATE, so on a - # first run the table does not exist yet: with the detection bypassed it fell through to the - # writer and stamped a stub table with empty fields, clobbering the creation that was the whole - # point of the call (the owner stamp and the flow tag went with it). A table that does not - # exist, or that is not an Instagram profile table, has nothing to migrate. - # ⭐ WAVE 32 · T42 — the CONTRACT set, not the detector's. This loop retypes columns, stamps - # `platform` and drops retired keys, i.e. it edits the schema — so it answers to the owner's - # rule about which databases Instagram may write into, exactly like `_ig_schema_contract` at - # the bottom of this function. `_ig_profile_tables` stays the wider question and keeps its own - # callers (`discover_default_table`, `backfill_corpus_snapshots`, the detector's NC). - detected = set(_ig_contract_tables(rt)) - targets = ([only] if only in detected else []) if only else sorted(detected) - # ⭐⭐ WAVE 31 · T30 — `only` NOW NARROWS THE WHOLE FUNCTION, AND IT DID NOT. - # - # ⛔ THE BUG WAS THAT `only` NARROWED THE LOOP ABOVE AND NOTHING ELSE. The tenant-wide passes - # at the bottom — two full `_ig_schema_contract` + `_reconcile_ig_graph_fields` rounds and two - # backfills — ignore `only` entirely, so `migrate_ig_tables(only=X)` paid a whole-tenant - # Instagram reconciliation no matter what X was. And `ut_ensure` calls it that way on EVERY - # ensure whose field set intersects `PRESET_PROFILE_KEYS` — which TikTok's profile schema does, - # because the two networks share column names. - # MEASURED on the wave-30 fixture, `only="ut_tt_profile"`: `_ig_profile_tables` returns `[]`, - # `stats` comes back completely empty and ZERO store writes happen — at a cost of **10 - # whole-document reads**, each a `Store.get` deep copy (documented ceiling 35.8 MB / ~1.4 s). - # Across one save's four `ut_ensure` calls that was **40 of the 41** reads behind owner item 7's - # *"it just say Saving... and takes a long time"*. The migration was not slow; it was running at - # all. - # ⚠ THE COMMENT ONE SCREEN DOWN IS WHY THIS WENT UNSEEN FOR THREE WAVES: *"Both are guarded on - # 'is there anything to do' and cost a dict scan in the steady state, which is what makes them - # safe on a function that rides every `ut_ensure`."* That guard is on the WRITE. The read was - # never guarded, and a scan of a freshly deep-copied 35 MB document is not a dict scan - # [[read-a-gates-predicate-for-what-it-excludes]]. - # ⛔ SCOPED TO THE PROVEN-EMPTY CASE ONLY, deliberately. When `only` IS a detected profile table - # the function is unchanged, tenant-wide passes included — that is the Instagram path every - # migration check in `verify_automation` exercises. And a caller passing no `only` still gets - # the full sweep. The claim being made here is narrow and checkable: *a table that is not in the - # Instagram graph has no Instagram migration*, which is the same thing the `detected` test above - # already decided one line earlier and then threw away. - # ⛔ AND IT IS NOT A LEND. Lending one snapshot through the rest of this function would be the - # obvious next fix and it is WRONG: the second contract pass exists precisely because it must - # read what the first pass WROTE (see its own note below), so a read-write-read sequence cannot - # answer out of one snapshot without silently un-fixing that idempotency. - if only and not targets: - return stats - for table_key in targets: - tbl = ut_get(rt, table_key) or {} - fields = [dict(f) for f in tbl.get("fields") or []] - rows = {rid: dict(r) for rid, r in (tbl.get("rows") or {}).items()} - # ⭐ THE IDEMPOTENCY KEY. Only a column whose STORED type is still the old one is touched, - # so the cell conversions below can never run twice on the same value. - stale = [f for f in fields - if f.get("key") in want and f.get("type") != want[f["key"]] - and f.get("type") in ("text", "", None)] - stale_keys = {f["key"] for f in stale} - has_platform = any(f.get("key") == "platform" for f in fields) - needs_stamp = [rid for rid, r in rows.items() if not str(r.get("platform") or "").strip()] - # ⭐⭐ 2026-08-07 (owner ruling) — MAKE THE HANDLE THE PRIMARY COLUMN AND THE BINDING. - # `ut_ensure` merges by KEY and never updates a field that already exists, so declaring the - # two keys on `PRESET_PROFILE_FIELDS` reaches NEW tables only. Every table already in - # production needs them stamped here, and that is the whole reason this branch exists. - # - # ⚠ IDEMPOTENT BY READING THE STORED DECLARATION, never by a marker column — W26's rule, - # and it is free here: re-stamping a boolean and a two-key dict is a no-op by nature, which - # is exactly why these are safe to run on every write where a value conversion would not be. - pin_f = next((f for f in fields if f.get("key") == PRESET_PINNED_KEY), None) \ - if PRESET_PINNED_KEY else None - flag_f = next((f for f in fields if f.get("key") == PRESET_FLAG_KEY), None) \ - if PRESET_FLAG_KEY else None - # ⛔ AT MOST ONE PROFILE COLUMN PER TABLE is the law `user_tables` enforces at both write - # doors, and a migration is not exempt from it. If somebody has already flagged another - # column on this table, stamping the handle too would leave the table in a state its own - # validator refuses — and `_profile_of` returns the FIRST match, so the enrich action would - # silently bind to whichever came first in the field list. Leave it alone and pin only. - other_profile = next((f for f in fields if f is not flag_f - and isinstance(f.get("profile"), dict)), None) - want_pin = pin_f is not None and pin_f.get("pinned") is not True - want_flag = (flag_f is not None and other_profile is None - and not isinstance(flag_f.get("profile"), dict)) - vestigial = _vestigial_name_field(fields, rows) - retired = {f.get("key") for f in fields} & {"posts", "post_hashtags"} - if _retired_tracked_field(fields) is not None: - retired.add(TRACKED_KEY) - orphan_tracked = _orphan_tracked_rows(fields, rows) - if (not stale_keys and has_platform and not needs_stamp - and not want_pin and not want_flag and vestigial is None and not retired - and not orphan_tracked): - continue - stats["tables"] += 1 - if want_pin: - pin_f["pinned"] = True - stats["pinned"] += 1 - if want_flag: - flag_f["profile"] = {"source": PROFILE_SOURCE_IG} - stats["flagged"] += 1 - if vestigial is not None: - # ⚠ THE CELLS GO WITH THE COLUMN. A row dict keeping a `name` key whose field no longer - # exists is invisible everywhere except the next export, where it reappears as a column - # nobody declared. `_vestigial_name_field` has already proven every one of them blank. - fields = [f for f in fields if f is not vestigial] - for r in rows.values(): - r.pop("name", None) - stats["droppedName"] += 1 - if retired: - fields = [f for f in fields if f.get("key") not in retired] - for r in rows.values(): - for key in retired: - r.pop(key, None) - stats["droppedRecentPosts"] += int("posts" in retired) - stats["droppedPostHashtags"] += int("post_hashtags" in retired) - stats["droppedTracked"] += int(TRACKED_KEY in retired) - for rid in orphan_tracked: - # D-74: a tick whose column was already gone. Counted apart from the column drop - # because they answer different questions — "how many tables still declared it" and - # "how many rows were still carrying it after the declaration went". - rows[rid].pop(TRACKED_KEY, None) - stats["droppedTrackedCells"] += 1 - - for key in stale_keys: - conv = _MIGRATE_CONVERT.get(key) - if not conv: - continue # int/checkbox already store the right text shape - for r in rows.values(): - if str(r.get(key) or "").strip(): - new = conv(r[key]) - # "" from a converter means UNREADABLE, and the cell keeps its original text. - # A migration that empties a cell is indistinguishable from one that moved it. - if new: - r[key], stats["converted"] = new, stats["converted"] + 1 - for f in stale: - f["type"] = want[f["key"]] - stats["retyped"] += 1 - for rid in needs_stamp: - rows[rid]["platform"] = PLATFORM_INSTAGRAM - stats["stamped"] += 1 - if not has_platform: - fields = [dict(f) for f in CANDIDATE_FIELDS if f["key"] == "platform"] + fields - rows, merged = _merge_candidates(rows) - stats["merged"] += merged - - def _up(cur, _f=fields, _r=rows): - cur = cur or {} - t = dict(cur.get(table_key) or {}) - t["fields"], t["rows"] = _f, _r - cur[table_key] = t - return cur - - rt.update(UT_STORE_KEY, _up) - # ⭐⭐ WAVE 33 (D-187) — A SINGLE-TABLE CALL STOPS PAYING FOR A TENANT-WIDE RECONCILIATION. - # - # W31-T30 closed HALF of this: `only` naming a table outside the Instagram graph returns early - # (`if only and not targets` above). What it left is the case the row is actually about — `only` - # naming a table that IS in the graph, i.e. **every save an Instagram tenant makes**. The four - # passes below are tenant-wide by construction: two `_ig_schema_contract` + - # `_reconcile_ig_graph_fields` rounds and two backfills, each walking the whole document, none - # of them reading `only`. So one `ut_ensure` on one table paid a whole-tenant Instagram - # reconciliation, and `ut_ensure` runs several times per save. - # - # ⛔ THE ROW'S EXIT OFFERS TWO BRANCHES AND THIS TAKES THE FIRST ONE — *"the tenant-wide passes - # are reachable from a scheduled/boot caller"* — because they ARE, and were before this change: - # `routes_tables.py:882` (`_ig_forward`) calls `migrate_ig_tables(rt)` with NO `only`, once per - # tenant per process, guarded by `_IG_FORWARDED`. That is the right home for a whole-tenant - # repair: once, off the save path, rather than on every ensure. - # ⚠ THE SECOND BRANCH — "bound them by an anything-to-do test guarded on the READ" — is NOT - # available here and the comment above says why: the second contract pass exists to read what - # the first pass WROTE, so it cannot be answered out of one snapshot without silently - # un-fixing the idempotency everything else depends on. - # ⛔ NARROW AND CHECKABLE, like its W31-T30 sibling: a call that NAMES ONE TABLE does that - # table's migration. A call that names none still does the whole sweep, unchanged, and every - # tenant-wide check in `verify_automation` goes through that path. - if only: - return stats - # The profile loop owns value conversions and deduplication. The schema pass owns the - # declarations across ALL related databases and is safe only after those conversions, because - # the stored old type is the idempotency key for unit-changing values such as engagement. - wanted, drops = _ig_schema_contract(rt) - stats["schema"] = int(_reconcile_ig_graph_fields(rt, wanted, drops)) - # ⛔⛔ A SECOND CONTRACT PASS, AND IT IS NOT BELT-AND-BRACES — WITHOUT IT THIS FUNCTION IS NOT - # IDEMPOTENT, which is the one property everything else here is built on. - # - # The derived overlay (`_derived_profile_columns`) decides by COMPARING the profile cell to the - # fold of the series. The pass above REPAIRS THE SERIES — it retypes the child columns and runs - # `_IG_RETYPE_CONVERTERS`, which rewrites `avg_engagement` from the vendor's 0-1 fraction to our - # 0-100 percent. So on a tenant whose children are stale, the first proof compares a repaired - # profile cell against an UNREPAIRED observation, finds them different, and declines to convert - # a column that is in fact perfectly derivable. The next call then converts it — and - # `verify_automation`'s "a SECOND pass is a no-op" check went red, which is exactly what that - # check is for. MEASURED on the wave-26 fixture: 24 columns converted on pass 2, none on pass 1. - # - # ⚠ The module already knew this shape one level up — the profile loop's own note says the - # schema pass "is safe only after those conversions, because the stored old type is the - # idempotency key". Same argument, one table further down. - # ⚠ CHEAP WHEN THERE IS NOTHING TO DO: on a current tenant the contract recomputes to the same - # declarations and `_reconcile_ig_graph_fields` writes nothing. Twice, not looped: the repair is - # what moves the data, and it has already run. - wanted2, drops2 = _ig_schema_contract(rt) - stats["schema"] = int(bool(stats["schema"] - | int(_reconcile_ig_graph_fields(rt, wanted2, drops2)))) - # ⭐⭐ 2026-08-10 — THE TWO VALUE BACKFILLS, and they run AFTER the schema pass on purpose: - # `backfill_post_snapshot_grain` writes into columns that pass has just declared, and running - # it first would find nothing to fill and report a truthful zero about the wrong moment. - # Both are guarded on "is there anything to do" and cost a dict scan in the steady state, which - # is what makes them safe on a function that rides every `ut_ensure`. - stats["series"] = backfill_corpus_snapshots(rt, log=log) - stats["postGrain"] = backfill_post_snapshot_grain(rt, log=log) - if stats["tables"] or stats["schema"] or stats["series"] or stats["postGrain"]: - log(f"[ig-migrate] {stats}") - return stats - - -# ── WAVE 25 · C6 (owner ruling R5) — SEEDING A SEARCH FROM A VIEW OR COHORT. ────────────────── -# "Find me more accounts like the ones in this view." The server reads the seed rows, ranks their -# SHARED characteristics, and FILLS IN the discovery conditions — visible and editable, never -# hidden (R5). -# -# ⛔ THE RANKING IS RESTRICTED TWICE, AND BOTH RESTRICTIONS ARE MEASURED RATHER THAN CHOSEN: -# 1. R5 restricts it to `BD_POPULATED_FIELDS` — a characteristic extracted from a field the -# corpus barely populates produces a filter that returns nothing and looks exactly like -# "no such accounts exist" (the trap `BD_FILTER_LEAD` already exists to warn about). -# 2. A derived predicate must NARROW, or `narrowing_refusal` refuses to save it. The -# intersection of the two sets is MEASURED at exactly eight fields — `account`, `biography`, -# `external_url`, `fbid`, `full_name`, `id`, `profile_name`, `profile_url` — and each of them -# narrows under its own `default_operator`, asserted in the gate rather than assumed. -# -# ⭐ AND FOUR OF THOSE EIGHT CANNOT GENERALISE, which is the part worth stating: `account`, `id`, -# `fbid` and `profile_url` are IDENTITIES. A filter derived from them re-finds the seed rows and -# nothing else — a search that returns what you gave it, at full price. So the ranker reads only -# the three that describe a KIND of account: shared words in the bio, a shared domain in the link, -# and shared words in the name. -SEED_SOURCES = ("view", "cohort") -SEED_MAX_ROWS = 200 # bounded and DISCLOSED in `basis.rows` — never a silent [:N] -SEED_MAX_PREDICATES = 3 # a filter of ten derived guesses is not a filter -SEED_MIN_COVERAGE = 0.5 # a "shared" characteristic under half the rows share is not shared -SEED_MIN_ROWS = 2 # one row has nothing to share WITH -#: ⛔ AND A COVERAGE BAR ALONE IS NOT ENOUGH, which the gate caught rather than review: at two -#: seed rows, `coverage >= 0.5` is satisfied by a word appearing in exactly ONE of them. So every -#: bio word in a two-row seed qualified as "shared", and the surface would have offered a -#: characteristic derived from a single record as the thing those records have in common. A -#: shared characteristic needs at least two rows to be shared BY — n=1 dressed as a pattern is -#: the fabrication this module refuses everywhere else. -SEED_MIN_ROWS_SHARING = 2 -#: Which ROW column feeds which VENDOR field. The row keys are the C1 preset set's; the vendor -#: names are `BD_FILTER_FIELDS`'. Two vocabularies for one thing, so the mapping is written down. -SEED_FROM_ROW = (("bio", "biography"), ("external_url", "external_url"), - ("full_name", "full_name")) -#: Words that are shared by everything and therefore discriminate nothing. Deliberately SHORT — -#: an aggressive list would silently drop a real signal, and the coverage bar already removes most -#: noise. Anything ≤2 characters is dropped by length instead of by enumeration. -SEED_STOPWORDS = frozenset({ - "the", "and", "for", "with", "you", "your", "our", "are", "not", "all", "any", "from", - "this", "that", "have", "has", "was", "com", "www", "http", "https", "out", "new", "more", - "who", "how", "その", "инст"} | {"official", "welcome", "contact", "email", "dm", "link"}) - - -def _seed_tokens(text): - """One cell → the distinct words worth matching on. Lowercased, de-punctuated, ≥3 chars.""" - words = re.findall(r"[a-z0-9]{3,}", str(text or "").lower()) - return {w for w in words if w not in SEED_STOPWORDS} - - -def _seed_domain(url): - """A link-in-bio → its registrable-ish host, or ''. `linktr.ee/x` and `linktr.ee/y` share a - domain and that IS the shared characteristic; the path is the thing that differs.""" - raw = str(url or "").strip() - if not raw: - return "" - host = (urlparse(raw if "//" in raw else "https://" + raw).hostname or "").lower() - return host[4:] if host.startswith("www.") else host - - -def seed_predicates(rows): - """R5: seed rows → `(derived_predicates, basis)`. PURE — no store, no vendor, no network. - - `basis` is what makes the offer honest rather than magic: how many rows were read, which - characteristics were extracted, and **the MEASURED coverage of each**, so a weak signal reads - as weak instead of being presented with the same confidence as a strong one. - - ⛔ EVERY DERIVED PREDICATE IS ASSERTED AGAINST `narrowing_refusal` BEFORE IT IS OFFERED - (HARD RULE 11). A suggestion the product's own save door then refuses is the post-W24 - default-versus-guard defect rebuilt — and it would be worse here, because the user did not - type it: they would be told their own product's suggestion is invalid. - - ⚠ AN EMPTY ANSWER IS A LEGITIMATE ANSWER. When the seed rows share nothing above the coverage - bar, `derived` is `[]` and `basis` says why. An honest nothing beats a filter that looks - plausible, returns zero rows, and reads as "no such accounts exist". - """ - rows = [r for r in (rows or []) if isinstance(r, dict)] - total = len(rows) - basis = {"rows": total, "fields": [], - # R5's second lane, reported as the MEASUREMENT it is rather than sold as a feature. - # `related_accounts` is 22% populated corpus-wide (D-25) and is not part of the C1 - # preset set, so on our own seed rows it is usually simply absent — which the surface - # must SAY, or a lane that adds nothing reads as a lane that failed. - "related": {"tried": total, - "found": sum(1 for r in rows - if str(r.get("related_accounts") or "").strip())}, - "note": ""} - if total < SEED_MIN_ROWS: - basis["note"] = ("a seed needs at least two records. One record has nothing to share " - "with anything") - return [], basis - ranked = [] - for row_key, vendor_field in SEED_FROM_ROW: - counts = {} - for r in rows: - vals = ({_seed_domain(r.get(row_key))} if vendor_field == "external_url" - else _seed_tokens(r.get(row_key))) - for v in vals: - if v: - counts[v] = counts.get(v, 0) + 1 - for value, n in counts.items(): - cov = n / total - if cov >= SEED_MIN_COVERAGE and n >= SEED_MIN_ROWS_SHARING: - ranked.append({"name": vendor_field, "value": value, "coverage": round(cov, 3), - "rows": n}) - # Strongest first; ties broken by the LONGER value, which is the more specific one. - ranked.sort(key=lambda f: (-f["coverage"], -len(f["value"]), f["name"])) - derived, seen_fields = [], set() - for f in ranked: - if len(derived) >= SEED_MAX_PREDICATES: - break - # One predicate per FIELD: two `biography includes` rows AND-ed together narrow to the - # accounts carrying both words, which is a much smaller search than the seed implies. - if f["name"] in seen_fields: - continue - p = {"name": f["name"], "operator": default_operator(f["name"]), "value": f["value"]} - if not predicate_narrows(p): - continue # cannot be offered; it would be refused at the save door - seen_fields.add(f["name"]) - derived.append(p) - basis["fields"].append({"name": f["name"], "label": field_label(f["name"]), - "value": f["value"], "coverage": f["coverage"]}) - if derived and narrowing_refusal(derived, "and"): - # Belt AND braces: the per-predicate check above should make this unreachable, and it is - # asserted in the gate. If the LAW ever changes shape, this fails closed with an honest - # empty rather than offering something the save door will reject. - basis["note"] = ("the shared characteristics found are not specific enough to search on " - "add a condition of your own") - return [], basis - if not derived: - basis["note"] = basis["note"] or ( - "these records share no bio words, link domain or name in common. Nothing could be " - "derived, so the conditions are yours to write") - return derived, basis - - -def seed_rows(rt, table_key, view_id=""): - """The rows a seed source selects. `(rows, problem)` — bounded, and the bound is DISCLOSED by - `basis.rows` rather than silently applied ([[no-unverifiable-aggregates]]).""" - t = ut_get(rt, str(table_key or "")) - if t is None: - return [], f"{table_key!r} is not a database in this workspace" - rows = list((t.get("rows") or {}).values()) - if str(view_id or "").strip(): - # ⛔ THE SAME RESOLVER `enters_view` USES, not a second one. A seed that selected a - # different row set from the view it names would be describing a view nobody has. - tree, _fields, problem = view_filter(rt, table_key, view_id) - if problem: - return [], problem - rows = [r for r in rows if lane_match(tree, r)] - return rows[:SEED_MAX_ROWS], "" - - -def automation_tables(defn): - """Every `ut_*` table THIS automation writes into — its config target plus every - `create_record` action's table, deduped, order preserved. - - ⚠ THE SAME TWO AUTHORITIES `observed_category_tables` READS, narrowed to one definition. - Split out rather than parameterised because the two questions are different: that one asks - "where could a Category value be, anywhere in this tenant" and is deliberately generous; this - one asks "which rows has THIS automation already found" and must not reach into a table it - does not write, or a nightly search would start excluding another automation's finds. - """ - keys, seen = [], set() - cfg = (defn or {}).get("config") or {} - for raw in [cfg.get("targetTable"), - *(((a or {}).get("config") or {}).get("table") - for a in walk_actions(((defn or {}).get("flow") or {}).get("actions")))]: - k = str(raw or "").strip() - if k.startswith(UT_PREFIX) and k not in seen: - seen.add(k) - keys.append(k) - return keys - - -def candidate_key(platform, handle): - """The discovery upsert identity: `(platform, handle)`, as one string. - - ⭐ WAVE 29 — LIFTED OUT OF `run_discover_instagram`, where it was a closure, because a second - runner now needs the same identity. Two closures computing "the same" key is how one of them - grows a different default for a blank `platform` and the two networks quietly start sharing - rows — which is the exact data-loss shape wave 26's R4 added `platform` to prevent. - - ⚠ A BLANK PLATFORM DEFAULTS TO INSTAGRAM, and that is a FACT about the stored data rather than - an assumption: every row written before wave 26 came from the Instagram runner. Blank-keyed - rows would fail to match their own re-find and duplicate the whole table on the next run. - """ - return f"{str(platform or PLATFORM_INSTAGRAM).strip()}\n{str(handle or '').strip()}" - - -def already_found_handles(rt, defn, cap=None): - """The handles this automation has already written, for the vendor-side exclusion (item 7). - - ⭐⭐ THE POINT IS THE BILL, not the row count. Discovery upserts by handle, so re-finding a - profile has always been harmless — and never free: the vendor bills for every record it - returns, so a nightly search over stable keywords pays again, in full, for the same accounts - every night and reports them as `seen_again`. MEASURED shape of the waste: nurilab's beauty - scout re-found its whole result set on every run. This is the list that stops it, sent as one - `not_in` the vendor evaluates BEFORE billing. - - ⚠ SCOPED TO THIS AUTOMATION'S OWN TABLES. Excluding handles another automation found would - hide profiles this one has never seen — cheaper, and wrong: two scouts with different keywords - are two questions, and one must not answer with "somebody already looked at that". - """ - cap = BD_EXCLUDE_MAX if cap is None else int(cap) - out, seen = [], set() - for table_key in automation_tables(defn): - for row in ((ut_get(rt, table_key) or {}).get("rows") or {}).values(): - h = str((row or {}).get("handle") or "").strip().lstrip("@").lower() - if h and h not in seen: - seen.add(h) - out.append(h) - if len(out) >= cap: - return out - return out - - -def observed_category_tables(rt): - """Every table a Category value could have been written into, DERIVED from the automations - this tenant actually has — the default discovery table and the snapshot table are the FLOOR, - not the list. - - ⭐ Two authorities, because wave 24 and wave 25 each moved where the target is declared: - `config.targetTable` (the discovery runner's own write) and any `create_record` action's - `config.table` (R2 — authoritative since wave 25, and the field the Builder's Database picker - writes). Both are read, deduped, order preserved so the defaults come first. - """ - keys, seen = [], set() - - def _add(k): - k = str(k or "").strip() - if k and k.startswith("ut_") and k not in seen: - seen.add(k) - keys.append(k) - - _add(DISCOVER_TABLE) - _add("ut_ig_snapshots") - try: - for defn in (all_definitions(rt) or {}).values(): - cfg = (defn or {}).get("config") or {} - _add(cfg.get("targetTable")) - for act in walk_actions(((defn or {}).get("flow") or {}).get("actions")): - _add(((act or {}).get("config") or {}).get("table")) - except Exception: # noqa: BLE001 - # A definition set that cannot be read costs the DERIVED lanes and keeps the defaults — - # the same posture as the master lane below: degrade to less vocabulary, never to an error. - pass - return keys - - -def observed_categories(rt, limit=60): - """⭐ DEBT D-59 — the Category values WE HAVE ACTUALLY SEEN, for a combobox that still accepts - free text. - - D-59's exit condition, verbatim: *"EITHER derive the options from values we have actually seen - (the `category` column on `ut_ig_candidates` + the master store, offered as a combobox that - still accepts free text), OR buy a sample large enough to enumerate the real vocabulary. **Not**: - transcribe a published taxonomy and hope it lines up."* This is the first branch. - - ⛔ WHY A PUBLISHED TAXONOMY WOULD HAVE BEEN WORSE THAN NO DROPDOWN. A filter on a value the - corpus does not use returns zero rows and looks exactly like an honest "no such accounts - exist" — so a picker built from Instagram's own category list would mislead precisely when it - looked most authoritative. Every option here has been observed on a real row, and each carries - its COUNT so a value seen once reads differently from one seen forty times. - - ⚠ IT STAYS A COMBOBOX. These are the values we have seen, not the values that exist — a - control that refused anything else would be a second, quieter version of the same lie. - """ - seen = {} - - def _eat(rows, key): - for r in rows or []: - v = " ".join(str((r or {}).get(key) or "").split())[:60] - if v: - seen[v] = seen.get(v, 0) + 1 - - # ⛔ THE TABLES ARE DERIVED, NOT NAMED — and this is a REGRESSION BY OMISSION that shipped - # green. The two constants below were the whole list, which was correct until **wave 25 R2 - # made the Create record action's `config.table` AUTHORITATIVE**: from that ruling on, a - # discovery automation writes wherever the user pointed it, and `ut_ig_candidates` is merely - # the DEFAULT nobody keeps. MEASURED on nurilab 2026-08-06 — `ut_ig_candidates` held 0 rows - # while the tenant's two real target tables held 20 each, so this function honestly reported - # `{options: []}` and the combobox it feeds had nothing to offer. A vocabulary harvester that - # names its sources goes quietly empty the moment the product lets a user choose one. - for key in observed_category_tables(rt): - _eat(((ut_get(rt, key) or {}).get("rows") or {}).values(), "category") - try: - # ...and the PLATFORM master, which is the whole point of pooling it: a tenant with three - # candidates still gets a vocabulary drawn from every profile the platform has captured. - import ig_master - if ig_master.configured(): - # ⚠ `_handle().get(SNAP_BUCKET)` — a dict of rows keyed by id. NOT `_upsert()`, which - # returns the upsert FUNCTION; calling that and reading `.get(...)` off it answers - # None, so the master lane would have degraded to nothing INSIDE the except below and - # this would have shipped as a silent no-op with the gate green. Verified against - # `ig_master.series_for`, which reads the same bucket the same way. - _eat((ig_master._handle().get(ig_master.SNAP_BUCKET) or {}).values(), "category") - except Exception: # noqa: BLE001 - # A master that cannot be read costs the tenant its own values and nothing else. - pass - return [{"value": v, "count": n} - for v, n in sorted(seen.items(), key=lambda kv: (-kv[1], kv[0]))[:limit]] - - -def preset_plan(rt, table_key): - """C1 / R2b: the preset set diffed against ONE database's CURRENT fields. - - `{table, fields: [{key, label, type, present}], willUse: [...], willCreate: [...]}` - - ⛔ COMPOSED HERE, NOT IN THE CLIENT. The owner's ask is a config panel that says which columns - an automation will USE and which it will CREATE — and the only thing that can answer it is - whatever holds both lists. A client that diffed the preset set against a table's fields would - be a second implementation of `ut_ensure`'s merge rule, free to disagree with the merge that - actually runs; it would be wrong precisely when the two lists differ, which is the only case - anybody is asking about. - - ⚠ A TABLE THAT DOES NOT EXIST IS NOT AN ERROR — it is the ordinary state at R10's spawn-on-save - moment, and the honest answer is "all of them will be created". `present` is a fact about the - table, so it reads False for every field rather than the call refusing. - - ⛔ 2026-08-07 — THE PROJECTION IS FOUR KEYS AND DELIBERATELY DROPS EVERY DECLARATION BAG - (`link`, `rollup`, `metric`, `profile`, `pinned`). Measured off the live Space the day the - relational pair shipped: `posts_link` and the four rollups arrive here with their bags EMPTY. - That is CORRECT for this payload — it answers "which columns will be used vs created", where - a name and a type are the whole question — and it is safe today because nothing RENDERS a - column from this list: the grid reads full field dicts from `/tables` (`scoped_pool` passes - `dict(f)` straight through), and the column itself is spawned server-side by `ut_ensure` from - `PRESET_PROFILE_FIELDS`, which carries the bag. - ⚠ **It stops being safe the moment somebody previews a rollup from this payload** — they would - render "Avg views - last 12 posts" configured by nothing. Read the field off `/tables`, or - widen this projection deliberately; do not assume a bag is here because the field has one. - """ - have = {str(f.get("key")) for f in ((ut_get(rt, str(table_key or "")) or {}).get("fields") - or [])} - fields = [{"key": f["key"], "label": f["label"], "type": f.get("type") or "text", - "present": f["key"] in have} - for f in PRESET_PROFILE_FIELDS] - return {"table": str(table_key or ""), - "exists": ut_get(rt, str(table_key or "")) is not None, - "fields": fields, - "willUse": [f for f in fields if f["present"]], - "willCreate": [f for f in fields if not f["present"]]} - - -def clean_predicates(raw, operator="and", kind=""): - """Validate a discovery filter into the vendor's shape. Returns `(predicates, error)`. - - Refuses rather than coerces: a predicate naming a field the API rejects comes back as a - 400 with the field named, because the alternative — dropping it — turns "find me verified - accounts in Georgia" into "find me any account" and bills for the difference. - - ⭐ WAVE 32 · T46 (D-167) — `kind` NARROWS THE VOCABULARY TO THE PLATFORM'S. Default `""` keeps - Instagram's 21 names, so every existing caller and every stored Instagram automation is - unchanged; a `discover_tiktok` is validated against the 5 names its corpus actually has. The - refusal it produces is the same sentence, listing the platform's own fields — which is the - difference between a search that says why it cannot be built and one that is built, paid for, - and comes back empty. - """ - fields, _lead = filter_fields(kind) - out = [] - for p in (raw or [])[:12]: - if not isinstance(p, dict): - continue - name = _s(p.get("name") or p.get("field"), 60).strip() - op = _s(p.get("operator") or p.get("op"), 20).strip() - if name not in fields: - return None, ("that field cannot be searched. The searchable ones are: " - + ", ".join(field_label(f) for f in fields)) - # ⭐ PER-FIELD, not the global list. `is_business_account >= 3` used to validate cleanly - # because every operator was legal on every field; a yes/no column offering "at least" - # is a question with no meaning that the vendor is nevertheless asked. - allowed = ops_for(name) - if op not in allowed: - return None, (f"{field_label(name)} cannot be asked {BD_OP_LABELS.get(op, op)!r}. " - "it takes: " - + ", ".join(BD_OP_LABELS.get(o, o) for o in allowed)) - entry = {"name": name, "operator": op} - if op not in BD_NULLARY_OPS: - raw_v = p.get("value") - # ONE value or SEVERAL — several is the "contains any of" shape (owner item 3), and - # it is stored as a list rather than as N sibling predicates so the row a person sees - # and the row that is stored are the same thing. - vals = raw_v if isinstance(raw_v, list) else [raw_v] - vals = [v.strip() if isinstance(v, str) else v for v in vals] - vals = [v for v in vals if not (v is None or (isinstance(v, str) and not v))] - if not vals: - return None, f"give {field_label(name)} something to compare against" - if len(vals) > 1 and op not in BD_MULTI_VALUE_OPS: - return None, (f"{BD_OP_LABELS.get(op, op)!r} takes one value. " - f"{field_label(name)} can only be given a list with " - + ", ".join(BD_OP_LABELS[o] for o in ops_for(name) - if o in BD_MULTI_VALUE_OPS)) - if len(vals) > MAX_PREDICATE_VALUES: - return None, (f"{field_label(name)} takes at most {MAX_PREDICATE_VALUES} values " - "in one condition") - if field_kind(name) == "boolean": - ok = {o["value"] for o in BD_BOOLEAN_OPTIONS} - bad = [v for v in vals if str(v).lower() not in ok] - if bad: - return None, f"{field_label(name)} is answered Yes or No" - # ⛔ A REAL BOOLEAN, NOT THE STRING. MEASURED 2026-08-06: the filter API answers - # **400** to `{"name": "is_verified", "operator": "=", "value": "true"}` and - # accepts `True` (`snap_msh2rp4kh3v833q0u`). Nobody had ever sent one — the field - # was a free-text box until this wave, so the Yes/No dropdown that makes it easy - # to ask is also the thing that would have made every boolean condition fail. - # The dropdown's option VALUES stay "true"/"false" (a carries strings); + # the conversion belongs here, at the edge that talks to the vendor. + vals = [str(v).lower() == "true" for v in vals] + if field_kind(name) == "number": + try: + vals = [float(v) if "." in str(v) else int(v) for v in vals] + except (TypeError, ValueError): + return None, f"{field_label(name)} takes a number" + # A single value stays a SCALAR on the wire. The vendor has only ever been sent + # scalars for these operators; the list form is expanded at send time and the + # one-value case must not quietly start exercising an untested shape. + entry["value"] = vals[0] if len(vals) == 1 else vals + out.append(entry) + if not out: + # ⭐ WAVE 24 · AMENDMENT A2 — AN EMPTY FILTER IS INCOMPLETE, NOT WRONG, so it STORES. + # This used to refuse, which was right while a wizard collected the filters before the + # automation existed. C-TRIG law 1 inverts that: picking the `ig_profile_match` trigger + # CREATES the automation, and the filters are typed afterwards — so a save-time refusal + # here meant the trigger could never be picked at all. It is the A3 + # stored-inert-with-`configured:false` pattern this module already runs on. + # + # ⛔ NOTHING IS LOST AT THE MONEY DOOR, which is why this is safe rather than convenient: + # `run_discover_instagram` ALREADY calls `narrowing_refusal(preds)` before starting a + # search, deliberately, because "a stored config can predate the law, and the vendor + # bills for breadth whether the filter was saved yesterday or last month". The RUN is the + # wall. Every MALFORMED predicate above is still refused here, and the narrowing law + # below still refuses a non-empty filter that narrows nothing — only EMPTY changed. + return [], None + # C4 (wave 22): breadth is refused at WRITE time too — see `narrowing_refusal` for the + # measured hang this guards against. Same sentence at create/patch and at run. + guard_err = narrowing_refusal(out, operator, kind) or depth_refusal(out, operator) + if guard_err: + return None, guard_err + return out, None + + +def discover_estimate(records_limit): + """The cost preview a Find node shows BEFORE it runs. **SPEC, never measured** — see the + section header. Returned as structured data so the UI cannot accidentally drop the caveat.""" + n = max(0, int(records_limit or 0)) + return {"records": n, "usd": round(n * BD_RECORD_PRICE_SPEC, 4), + "unitUsd": BD_RECORD_PRICE_SPEC, "basis": "SPEC", + "note": "estimated from the published rate. An exact price is only known after a run" + "a price before a run, and this account's token cannot read a balance"} + + +def _candidate_row(row, stamp): + """One corpus row → a `ut_ig_candidates` row. None when it has no handle. + + ⛔ NEVER EMITS `found_count` — it is arithmetic the runner does against what is already + stored, so writing it here would reset the counter to 1 on every re-find. (It never emitted + `tracked` either; R6 deleted that column in wave 26.) + """ + if not isinstance(row, dict): + return None + handle = str(_first(row, "account", "username", default="") or "").strip() + if not handle: + return None + return { + # ⭐ R5 — half the identity. See `PLATFORM_INSTAGRAM`: this runner only ever reads + # Instagram, so it is a constant here rather than something derived from the row; the + # TikTok runner will stamp its own and the (platform, handle) key keeps the two apart. + "platform": PLATFORM_INSTAGRAM, + "handle": handle, + "profile_url": str(_first(row, "profile_url", "url", + default=f"https://www.instagram.com/{handle}/")), + "full_name": str(_first(row, "full_name", "profile_name", default="") or ""), + "followers": _s(_ig_int(_first(row, "followers"))), + "following": _s(_ig_int(_first(row, "following"))), + # ⭐ POPULATED ON CORPUS ROWS and null on scrape rows — the two paths differ, and this is + # the path where it carries values (measured on all five of the 2026-08-04 result set). + # ⚠ ×100 since wave 26 (amendment C1-a): the vendor's fraction is not our `pct`. + "avg_engagement": _pct100(_first(row, "avg_engagement", default="")), + "bio": _s(_first(row, "biography", "bio", default=""), 500), + "external_url": _s(_bd_first_url(_first(row, "external_url", "external_urls")), 300), + "verified": "1" if _first(row, "is_verified", default=False) else "", + "category": _s(_first(row, "category_name", "business_category_name", default="")), + # R3: a DAY, matching the `date` type the column now declares. The full-precision stamp + # still exists on the snapshot series, which is where a time axis belongs. + "last_found": _day(stamp), + } + + +# ── ⭐⭐ 2026-08-10 — DISCOVERY'S MISSING OBSERVATION ────────────────────────────────────────── +# +# ⛔ THE DEFECT, MEASURED ON NURILAB BEFORE ANY OF THIS WAS WRITTEN: 105 profiles, 104 carrying a +# `followers` number, and only 97 with a single row of history behind it. Seven accounts had a +# measurement nothing could re-derive, date or audit — `19,448 followers`, as of never, read via +# nothing. `_candidate_row` above writes six MEASUREMENTS onto a profile row (followers, +# following, engagement, verified, category, bio) and the discovery runner wrote no snapshot at +# all, so discovery was the one rung in this module that read numbers and recorded no observation. +# +# ⛔ IT IS NOT A "MISSING ENRICHMENT". Those seven rows are not waiting to be enriched — they hold +# real corpus numbers that are already on screen and already filterable. The gap is that the +# ENTITY row is the only copy, which is the exact arrangement R3's "one store for one series" law +# exists to forbid one level up. +# +# ⚠ A CORPUS ROW IS AN HONEST OBSERVATION, NOT A FAKE MEASUREMENT, and the two columns that make +# it honest already existed: `source` says **Discovery** (this was read off the vendor's +# pre-collected corpus, not measured for you) and `approx` is checked. Together they say "do not +# read this as an exact count taken at `pulled_at`" — which is the whole difference between +# recording what we know and inventing what we do not. +# +# ⭐ AND THE DATE GRAIN MAKES THE TIE-BREAK COME OUT RIGHT, which is worth stating because it +# looks like an accident: `pulled_at` here is a DAY (`last_found`), so `_sort_key` reads it as +# MIDNIGHT, while an enrichment on that same day carries a real timestamp. A `latest` rollup over +# the series therefore prefers the exact read over the corpus read whenever both happened on one +# day — the ordering you would have to hand-write, falling out of the grain. + +#: The `via` a DISCOVERY read reports. `PUBLIC_SOURCE` maps it to the word in the cell. +IG_VIA_DISCOVERY = "brightdata:discovery" + +#: The five keys a snapshot row owns about ITSELF. Everything else it carries is a measurement +#: copied off the profile row, and the list of those is DERIVED from `SNAPSHOT_FIELDS` rather +#: than typed out — a hand-list is what silently stops carrying the next promoted column. +_SNAPSHOT_OWN_KEYS = ("snapshot_key", "influencer_key", "pulled_at", "source", "approx") + + +def corpus_snapshot_row(row, day=""): + """ONE profile row read from the corpus → its `ut_ig_snapshots` observation. None if unusable. + + `day` overrides the row's own `last_found`, which is what the BACKFILL passes when a row's + only date is `first_found`. + + ⛔ IDEMPOTENT BY ITS KEY, never by a flag or a marker column. `snapshot_key` is + `@`, so re-running this over the same rows on the same day rewrites the same key + and `upsert_rows` merges it — the migration is safe to call on every write, which is the only + way it can be safe to call at all (W26's rule, and the same reason `migrate_ig_tables` keys + idempotency on the stored TYPE). + + ⚠ BLANKS ARE OMITTED RATHER THAN WRITTEN. `capture_rows` writes every key including the empty + ones because a paid pull's blank means "this rung did not read it" and the row is append-keyed. + Here the merge is the hazard instead: a second discovery on the same day returning a thinner + corpus record would otherwise ERASE what the first one learned. An absent key renders exactly + as an empty cell, so nothing is lost by leaving it out. + """ + handle = str((row or {}).get("handle") or "").strip().lstrip("@").lower() + if not handle: + return None + when = _day(day or (row or {}).get("last_found") or (row or {}).get("first_found") or "") + if not when: + return None + out = {"snapshot_key": f"{handle}@{when}", "influencer_key": handle, "pulled_at": when, + "source": _s(public_source(IG_VIA_DISCOVERY)), "approx": "1"} + for fd in SNAPSHOT_FIELDS: + key = fd["key"] + if key in _SNAPSHOT_OWN_KEYS: + continue + value = (row or {}).get(key) + if str(value or "").strip(): + out[key] = str(value) if key == "source_payload" else _s(value, 400) + return out + + +def append_ig_snapshots(rt, incoming, snap_key=IG_SNAPSHOTS_TABLE, log=print): + """Merge corpus observations into an EXISTING `ut_ig_snapshots`. Returns rows written. + + ⛔ IT NEVER CREATES THE TABLE, and that refusal is structural rather than cautious: the only + honest way to create it is `ensure_ig_graph`, which calls `ut_ensure`, which calls + `migrate_ig_tables` — and the BACKFILL caller is inside `migrate_ig_tables`. Reaching for the + creator from there is unbounded recursion. A tenant with no series store has nothing to + back-fill INTO; the forward path (`run_ig_discovery`) creates the graph properly and this + function then has somewhere to write. + """ + rows_in = [r for r in (incoming or []) if r] + if not rows_in: + return 0 + existing = ut_get(rt, snap_key) + if existing is None: + return 0 + merged, counts = upsert_rows(dict(existing.get("rows") or {}), rows_in, "snapshot_key", + cap=row_cap(snap_key)) + if counts["capped"]: + # D-11's law: a full append table means the SERIES has stopped growing, which is the one + # failure a chart cannot show you. + log(f"[aios-auto] corpus series: {counts['capped']} observation(s) refused by " + f"{snap_key}'s row cap") + written = counts["inserted"] + counts["updated"] + if not written: + return 0 + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + if cur.get(snap_key) is not None: + cur[snap_key]["rows"] = merged + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") + return written + + +# --------------------------------------------------------------------------------------------- +# RUNNERS +# --------------------------------------------------------------------------------------------- + +def cap_note(capped, missing=()): + """The sentence(s) a cap breach adds to a run summary. + + `capped` = `[(table_key, rows_lost), …]` — the ROW ceiling was hit. + `missing` = `[table_key, …]` — the TABLE ceiling was hit: the database could not + be created at all. + + ⛔ IT NAMES THE TABLE. "142 rows skipped" sends somebody to look at the source page; "the + ut_ig_post_snapshots database is FULL" sends them to the actual problem. A loud failure that + does not say WHERE is only marginally better than a silent one. + + ⛔ AND THE TABLE CEILING IS NOW AS LOUD AS THE ROW ONE (closes DEBT D-11). `ut_ensure` refused + SILENTLY at `MAX_UT_TABLES`: it returned a key for a database it had not created, the + runners' `if tgt is not None` write guard then skipped that bucket, and the run reported + success over a database that does not exist. Exactly the silent-stop failure the row cap + already had, one level up — and harder to spot, because the table is not there to look at. + """ + out = [f"⚠ the {k} database is FULL ({row_cap(k):,} rows). {n} row{'' if n == 1 else 's'} " + f"from this run were NOT written" for k, n in (capped or []) if n] + out += [f"⚠ the {k} database could NOT BE CREATED. This workspace is at its " + f"{MAX_UT_TABLES}-database limit, so nothing from this run reached it" + for k in (missing or [])] + return "; ".join(out) + + +def ut_missing(rt, *keys): + """Which of `keys` do NOT exist after an `ut_ensure` — the table-ceiling detector (D-11).""" + have = ut_all(rt) + return [k for k in keys if k and k not in have] + + +def run_scrape_db(rt, defn, username="automation", log=print, step=_no_step, rows=None): + """Automation #1 (R6): a public page → a blank database, re-runnable, UPSERT by key.""" + cfg = defn.get("config") or {} + fmap, key_field = cfg.get("fieldMap") or {}, cfg.get("keyField") + dry = bool(cfg.get("dryRun")) + # Per-NODE outcomes for the canvas. MEASURED as the run walks, never inferred afterwards from + # the rollup — inferring is exactly how a green dot ended up over an empty table (see the + # rollup comment in `run_field_instagram`). + steps = {"trigger": "ok", "fetch": "idle", "extract": "idle", "write": "idle"} + step(f"Fetching {urlparse(str(cfg.get('url') or '')).hostname or 'the page'}") + status, final, body = fetch(cfg["url"]) + if not (200 <= status < 300): + steps["fetch"] = "error" + return ("error", f"{cfg['url']} answered {status}. Nothing was written", + {"status": status}, [], steps) + steps["fetch"] = "ok" + soup = _soup(body) + if cfg.get("extract") == "jsonld": + blocks = jsonld(soup) + flat = [] + for b in blocks: + if isinstance(b, list): + flat.extend([x for x in b if isinstance(x, dict)]) + elif isinstance(b, dict): + items = b.get("itemListElement") + flat.extend([x for x in items if isinstance(x, dict)] if isinstance(items, list) + else [b]) + raw_rows = [{k: _scalar(v) for k, v in d.items()} for d in flat] + else: + found = tables(soup) + idx = max(0, min(int(cfg.get("tableIndex") or 0), len(found) - 1)) if found else 0 + raw_rows = [r for r in (found[idx] if found else []) if isinstance(r, dict)] + if not raw_rows: + steps["extract"] = "error" + return ("error", "the page parsed but the selected table had no rows", {"rows": 0}, [], + steps) + steps["extract"] = "ok" + step(f"Extracting {len(raw_rows)} row{'' if len(raw_rows) == 1 else 's'}") + + incoming = [] + for r in raw_rows: + mapped = {tgt: _s(r.get(src, ""), 500) for src, tgt in fmap.items()} + if any(v for v in mapped.values()): + incoming.append(mapped) + + fields = [field_def(tgt, src if len(src) <= 60 else src[:60]) + for src, tgt in fmap.items()] + # the key column leads, so the table opens on the identity it is upserted by + fields.sort(key=lambda f: 0 if f["key"] == key_field else 1) + label = cfg.get("targetLabel") or defn.get("name") or "Scraped table" + if dry: + # THE WRITE NODE IS OFF — read, compute, report, touch nothing. Not even `ut_ensure`, + # which would create the table: a dry run that leaves a new empty database behind is not + # a dry run. + table_key = ut_key_for(label, cfg.get("targetTable") or None) + else: + table_key = ut_ensure(rt, label, fields, username, key=cfg.get("targetTable") or None) + t = ut_get(rt, table_key) or {} + # ⛔ D-116 — THE ONE CALL SITE THAT NEEDS A PRECEDENCE RULE, because it is the one whose rows + # come from a CORPUS. Every other `upsert_rows` caller writes measurements or appends a series. + rows, counts = upsert_rows(t.get("rows") or {}, incoming, key_field, + cap=row_cap(table_key), protect=corpus_protect) + touched = [rid for rid, row in rows.items() + if str(row.get(key_field, "")) in {str(i.get(key_field)) for i in incoming}] + if not dry: + ut_write_rows(rt, table_key, rows) + counts["source_rows"] = len(incoming) + affected = touched[:200] + summary = (f"{counts['inserted']} new, {counts['updated']} updated, " + f"{counts['unchanged']} unchanged, {counts['orphans']} no longer on the page " + f"(kept)") + # ⛔ D-116 — SAY IT. A search that quietly declines to overwrite six measured cells is doing + # the right thing invisibly, which is indistinguishable from a corpus that happened to agree — + # and the next person to wonder why a number did not move has nothing to read. The observation + # still lands in the series either way; this sentence is about the SCALAR. + if counts.get("held"): + summary += (f", {counts['held']} measured value(s) kept over the corpus " + f"(an enrichment read them exactly; the search's own numbers are in the " + f"snapshot series)") + capped = [(table_key, counts.get("capped", 0))] + missing = [] if dry else ut_missing(rt, table_key) + counts["missing_tables"] = len(missing) + note = cap_note(capped, missing) + if note: + summary += f". {note}" + steps["write"] = "partial" + if dry: + summary = f"Test run. Nothing saved. Would have written: {summary}" + steps["write"] = "skipped" + elif not note: + steps["write"] = "ok" + state = "partial" if (counts.get("capped") or counts.get("skipped") or missing + or dry) else "ok" + log(f"[aios-auto] scrape_db {defn.get('id')} -> {table_key}: {summary}") + return (state, summary, counts, affected, steps) + + +def capture_rows(res, pulled): + """ONE pull → `(snapshot_row, post_identity_rows, post_metric_rows, comment_rows)`. + + ⭐ WAVE 25 (C4) — FACTORED OUT OF `run_field_instagram` SO THE ENRICH ACTION CAN REUSE IT + RATHER THAN FORK IT. C4's instruction is literal: "Reuse `pull_profile` and the write path — + do not fork them." A second copy of this mapping is the shape that goes wrong invisibly, + because the two copies would each be *plausible* and would disagree only on the rows a + particular rung happened to return. + + ⚠ THE TWO WRITE DISCIPLINES ARE OPPOSITE HERE, AND BOTH ARE DELIBERATE: + * A SNAPSHOT ROW IS APPEND-KEYED (`@`), so every key is written every + time and a blank is unambiguous — it means THIS PULL did not read it. + * AN IDENTITY ROW OMITS ITS BLANKS, because it is UPSERTED: `posted_at`/`caption`/`type` are + readable on some rungs and not others (the Bright Data PROFILE row carries post identity + with `datetime: None`, measured 24/24, while the engagement rung knows `date_posted`), and + `upsert_rows` writes exactly the keys it is handed — so sending "" would let a cheap run + ERASE what an expensive one learned. + * AN ENGAGEMENT SNAPSHOT IS APPENDED ONLY WHEN SOMETHING WAS MEASURED. The post series is + the append table that FILLS (maxPosts rows per profile per pull); with the paid engagement + rung OFF every one of those rows would carry three blanks — pure noise, eating a 200k + ceiling and drawing a chart of nothing. A row in a time series should mean "this was true + then"; a row meaning "nobody looked" belongs nowhere. + """ + prof = (res or {}).get("profile") or {} + snap_row = { + "snapshot_key": f"{prof.get('username')}@{pulled}", + "influencer_key": prof.get("username"), "pulled_at": pulled, + # ⛔ THE PUBLIC WORD, NOT THE VENDOR KEY — this cell is rendered in a grid column called + # "Read via". `res["via"]` keeps the real key for the call graph and the server log. + "source": _s(public_source((res or {}).get("via"))), "approx": _s(prof.get("approx")), + "bio": _s(prof.get("bio"), 500), + "external_url": _s(prof.get("external_url"), 300), + } + for fd in SNAPSHOT_FIELDS: + k = fd["key"] + if k not in snap_row: + value = prof.get(k) + # Profile history declares the same percentage dialect as the current Profile row. + # Converting only the latest projection left the append series at 0-1 while its field + # now said pct (0-100), so the two views of one measurement disagreed by 100x. + if k == "source_payload": + snap_row[k] = str(value or "") + else: + snap_row[k] = _s(_pct100(value) if k == "avg_engagement" else value, 400) + idents, metrics_rows, comment_rows = [], [], [] + for p in (res or {}).get("posts") or []: + ident = {"shortcode": p["shortcode"], "influencer_key": prof.get("username"), + "url": _s(p.get("url"), 300)} + for k, n in (("posted_at", 200), ("type", 200), ("caption", 800), + ("paid_partnership", 8), ("partner", 120), ("hashtags", 400), + ("alt_text", 400), ("tagged_location", 400)): + if p.get(k): + ident[k] = _s(p.get(k), n) + if p.get("source_payload"): + ident["source_payload"] = str(p["source_payload"]) + metrics = {k: p.get(k) for k in ("likes", "comments", "views", "plays")} + # ⭐⭐ 2026-08-07 — THE LATEST ENGAGEMENT VALUES, ONTO THE POST ROW ITSELF. + # + # This is what keeps a rollup at ONE HOP (Airtable's rule and ours): without it, "average + # views over the last 12 posts" would have to walk profile → posts → each post's most + # recent snapshot, and a two-hop rollup is a much larger feature with a much worse + # invalidation story. The post row now carries its own latest, exactly as the profile row + # carries LATEST + `enriched_at` while its series lives in `ut_ig_snapshots` (R3). + # + # ⛔ ONE STORE FOR ONE SERIES IS UNTOUCHED: `ut_ig_post_snapshots` below is still the + # authoritative engagement series and still gets its appended row. These cells are a + # projection of the row being appended in the same breath, never a second source. + # ⛔ A KEY IS WRITTEN ONLY WHEN THE VENDOR ANSWERED. `upsert_rows` merges, so an absent + # key leaves the previous pull's value standing — which is the correct behaviour for a + # LATEST column and the reason this cannot be a blanket `_s(...)` over all three: writing + # "" on a run that did not buy metrics would ERASE what an earlier paid run learned, the + # same failure the snapshot-append rule above is written against. + measured = {k: v for k, v in metrics.items() if v is not None and str(v).strip() != ""} + if measured: + ident.update({k: _s(v, 40) for k, v in measured.items()}) + # The stamp is what makes the numbers readable: a blank `plays` beside a + # `measured_at` of last week means "we looked and the vendor had nothing", and with + # no stamp it means that AND "we never looked", indistinguishably. + ident["measured_at"] = _day(pulled) + idents.append(ident) + if any(v is not None for v in metrics.values()): + metrics_rows.append({ + "post_snapshot_key": f"{p['shortcode']}@{pulled}", + "shortcode": p["shortcode"], "influencer_key": prof.get("username"), + "pulled_at": pulled, + # The two denormalised post facts (see `POST_SNAPSHOT_FIELDS`). Written from the + # SAME vendor record the identity row above is built from, so they cannot disagree + # with it on this pull. + # ⚠ BLANK IS EXPECTED AND IS NOT A BUG on the profile rung: `wave20-split` + # measured `datetime` as None on 24/24 posts from the Profiles dataset, so a pull + # that learns a post's engagement often does not learn its date in the same + # breath. `backfill_post_snapshot_grain` fills those from the post row, which by + # then may have learned it from a different rung. + "posted_at": _s(p.get("posted_at"), 200), "type": _s(p.get("type"), 200), + "likes": _s(metrics["likes"]), "comments": _s(metrics["comments"]), + "views": _s(metrics["views"]), "plays": _s(metrics["plays"]), + "source_payload": str(p.get("source_payload") or "")}) + for comment in p.get("embedded_comments") or []: + # The embedded shape often names only the commenter. Its parent Post is the + # authoritative owner, so bind every preview to this profile and post explicitly. + comment_rows.append({**comment, "shortcode": p["shortcode"], + "influencer_key": prof.get("username") or ""}) + comment_rows.extend((res or {}).get("comments") or []) + # The same comment can be returned as both `latest_comments` and `top_comments`, or by the + # optional full endpoint after an embedded preview. The canonical key is the one row truth. + unique_comments = {} + for row in comment_rows: + key = str((row or {}).get("comment_key") or "") + if key: + unique_comments[key] = {**(unique_comments.get(key) or {}), **row} + return snap_row, idents, metrics_rows, list(unique_comments.values()) + + +#: C4 — how ONE pulled profile becomes the C1 preset CELLS on the record being enriched. +#: `profile key -> preset column key`, so the mapping is a table rather than sixteen lines of +#: `row[...] = prof.get(...)` that a future field addition can silently miss. +#: ⚠ Keys absent from a pull are simply not written (R3: a blank preset cell means "this pull did +#: not read it", and `enriched_at` is what distinguishes that from "never enriched"). +PRESET_FROM_PROFILE = { + "username": "handle", "profile_url": "profile_url", "full_name": "full_name", + "followers": "followers", "following": "following", "avg_engagement": "avg_engagement", + "bio": "bio", "external_url": "external_url", "verified": "verified", + "category": "category", "posts_count": "posts_count", + "highlights_count": "highlights_count", "is_business": "is_business", + "is_professional": "is_professional", "ig_id": "ig_id", + # ⭐ 2026-08-07 — the promoted fields. A preset column with no row in this table is a column + # that can only ever be blank, so "make the preset fields populated" is THIS half of the + # owner's instruction and the field list is only the other half. + # ⚠ The two lists are held in step by a derived gate rather than by care: every + # `PRESET_PROFILE_KEYS` entry must either be written by this map or be explicitly declared + # as written elsewhere (`platform`, `enriched_at`, `posts` are stamped by `preset_cells`). + "business_category": "business_category", "is_private": "is_private", + "bio_hashtags": "bio_hashtags", + "pronouns": "pronouns", "profile_name": "profile_name", + "is_joined_recently": "is_joined_recently", "has_channel": "has_channel", + "partner_id": "partner_id", "external_url_title": "external_url_title", + "fbid": "fbid", "related_accounts": "related_accounts", + "country_code": "country_code", "source_payload": "source_payload", +} + +#: ⭐ 2026-08-07 — the preset keys `PRESET_FROM_PROFILE` deliberately does NOT carry, because a +#: different writer stamps them. DERIVED gates compare the two lists, and without this the gate +#: could only be written as "these three are fine" — a hard-coded exception list, which is the +#: shape [[gate-answers-the-wrong-question]] warns about. +#: ⚠ The five relational columns are written by `compute_relation_cells` on the tick, NOT by an +#: enrichment run — which is the whole point of them: they stay true when the LINKED table +#: changes, and a pull that touched no profile still updates a profile's post count. +PRESET_WRITTEN_ELSEWHERE = ( + "platform", "handle", "enriched_at", + "posts_link", "profile_snapshots_link", "post_snapshots_link", "comments_link", + "avg_views_12", "avg_plays_12", "avg_likes_12", "avg_comments_12", + "posts_captured", "profile_reads", "post_measurements_captured", "comments_captured", +) + + +#: A location guess needs at least this many GEOTAGGED posts to agree with. One post is not a +#: pattern and would render as 100% confident, which is the fabrication `SEED_MIN_ROWS_SHARING` +#: refuses on the discovery side for exactly the same reason. +LOCATION_MIN_POSTS = 2 + + +def _post_place(post): + """The city a post was tagged in, or ''. Reads the normalised field FIRST, the paid payload + second. + + ⛔ THE SECOND READ IS THE POINT AND IT COSTS NOTHING. `_bd_tagged_location` flattens the + vendor's `location` array into "Capanema, Pará, Brasil" and DISCARDS the rest — including the + `lat`/`lng`/`name` object Bright Data returns on some rows and the whole `location_details` + shape. The complete vendor row is retained in `source_payload` on the same record, already + paid for, so a post whose normalised field came back blank can still be read here. + ⚠ THE FIRST COMPONENT IS THE CITY. The vendor's array is ordered outward — city, region, + country — so the head is the narrowest thing it told us, and "Jakarta" is the answer to + "where is this person"; "Indonesia" mostly is not. + """ + if not isinstance(post, dict): + return "" + raw = str(post.get("tagged_location") or "").strip() + if not raw: + try: + payload = json.loads(post.get("source_payload") or "{}") + except Exception: # noqa: BLE001 + payload = {} + raw = str(_bd_tagged_location(payload) or "") if isinstance(payload, dict) else "" + return raw.split(",")[0].strip() + + +def location_guess(posts): + """`(city, confidence_pct)` from the places a creator's own posts were tagged in. + + `confidence` is the share of GEOTAGGED posts that agree on the modal city — deliberately not + the share of ALL posts, because a creator who geotags three of twelve is telling us about + three, and dividing by twelve would report a real signal as weak. How thin the evidence is + stays visible a different way: `LOCATION_MIN_POSTS` refuses to answer at all below two. + """ + places = [p for p in (_post_place(x) for x in posts or []) if p] + if len(places) < LOCATION_MIN_POSTS: + return "", None + counts = {} + for p in places: + counts[p] = counts.get(p, 0) + 1 + # ⚠ TIE-BROKEN BY FIRST APPEARANCE, never by sort order: two cities at 3/3 would otherwise be + # resolved alphabetically, which is an arbitrary answer wearing a confident number. + best = max(places, key=lambda p: (counts[p], -places.index(p))) + return best, round(100.0 * counts[best] / len(places), 1) + + +def preset_cells(res, pulled): + """C4/R3: one pull → the LATEST-value cells written onto the enriched record. + + ⛔ `enriched_at` IS ALWAYS WRITTEN when a pull succeeded, and it is the field that makes the + other fifteen readable: without it a blank `followers` means both "never enriched" and + "enriched in March and the vendor had nothing", and no cell on the row can tell them apart. + """ + prof = (res or {}).get("profile") or {} + cells = {} + for src, dest in PRESET_FROM_PROFILE.items(): + v = prof.get(src) + if v is not None and v != "": + # ⚠ WAVE 26 / C1-a — the ONE key whose value is not carried straight across. The + # vendor's engagement is a 0–1 fraction and our `pct` column is 0–100, so a + # straight-through copy here would write the same wrong number the discovery path + # used to write. Both writers convert; neither is the exception. + cells[dest] = (_pct100(v) if dest == "avg_engagement" else + (str(v) if dest == "source_payload" else _s(v, 500))) + # ⭐ R5 — stamped by the writer, not carried from the pull: this function only ever sees an + # Instagram profile. It matters on an ENRICH of a row somebody typed by hand, which may have + # arrived with no platform at all — and a blank half of the dedup key is how one account + # becomes two rows. + cells["platform"] = PLATFORM_INSTAGRAM + # ⭐ ITEM 16 — the residency guess, off posts this pull already paid for. Written only when + # there is one: a blank is "we could not tell", and overwriting a good guess from a run whose + # twelve posts happened to carry no geotag would make the column worse the more it ran. + place, confidence = location_guess((res or {}).get("posts")) + if place: + cells["location_guess"] = _s(place, 120) + cells["location_confidence"] = str(confidence) + # R3: `enriched_at` is a `date` column now. It answers "how stale is this number", which is a + # question in days; the full stamp keeps its precision on the snapshot series. + cells["enriched_at"] = _day(pulled) + return cells + + +#: Keys the TikTok SNAPSHOT row owns itself, so the copy loop below never overwrites them from the +#: profile map. Mirrors `_SNAPSHOT_OWN_KEYS` on the Instagram side. +_TT_SNAPSHOT_OWN_KEYS = ("snapshot_key", "influencer_key", "pulled_at", "source", "approx") + + +def tt_preset_cells(res, pulled): + """⭐⭐ WAVE 30 · T08 — one TikTok pull → the LATEST-value cells written onto the record. + + ⛔ THIS IS NOT `preset_cells` WITH A FLAG, AND THAT IS THE WHOLE DESIGN DECISION. Instagram's + `preset_cells` walks `PRESET_FROM_PROFILE` (an Instagram map), stamps + `cells["platform"] = PLATFORM_INSTAGRAM` UNCONDITIONALLY, and computes a location guess from + posts. Passing a TikTok row through it would stamp every TikTok creator as Instagram — which is + the identity half of W26's `(platform, handle)` key, so it would not merely mislabel a row, it + would MERGE two different people's accounts under one key. + ⭐ AND THE TIKTOK ROW DOES NOT NEED A MAP AT ALL. `connectors_tt.normalize_profile` already + emits our column names — it is the SAME function the discovery runner uses — so a scraped row + and a corpus row cannot disagree about which vendor key became which column. All this adds is + what the FETCH knows and the map cannot: when it was read, and by which route. + + ⚠ FILTERED TO THE DECLARED SCHEMA. `ut_write_rows`/`_clean_field` DROP an undeclared key with + no error and a successful-looking run (the defect `section_w29_tiktok_schema`'s emit-vs-declare + sweep exists for), so an unknown key is refused here where it is visible rather than swallowed + three layers down. + """ + prof = (res or {}).get("profile") or {} + declared = {f["key"] for f in TT_PROFILE_FIELDS} + cells = {k: v for k, v in prof.items() if k in declared} + # `source` is the ROUTE that answered, not the vendor's name — the same thing `via` carries on + # the Instagram snapshot, and the reason a stored measurement can always say how it was read. + if (res or {}).get("via"): + cells["source"] = _s((res or {}).get("via"), 60) + # R3, as on the Instagram side: a `date`, because "how stale is this number" is a question in + # days. Without it a blank `followers` means both "never enriched" and "enriched and empty". + cells["enriched_at"] = _day(pulled) + return cells + + +#: The metric columns a TikTok post SNAPSHOT carries. ⛔ DERIVED from the declaration, minus the +#: keys the snapshot owns itself — so adding a metric to `TT_POST_SNAPSHOT_FIELDS` starts being +#: captured, and adding an IDENTITY column to it never does. +_TT_PSNAP_OWN_KEYS = ("platform", "post_snapshot_key", "shortcode", "influencer_key", "pulled_at", + "post_link") +TT_PSNAP_METRIC_KEYS = tuple(f["key"] for f in TT_POST_SNAPSHOT_FIELDS + if f["key"] not in _TT_PSNAP_OWN_KEYS) + + +def tt_capture_rows(res, pulled): + """⭐⭐ WAVE 30 · T10 — one TikTok pull → `(post_identity_rows, post_metric_rows, comment_rows)`. + + The TikTok twin of `capture_rows`, and separate for the same reason `tt_preset_cells` is: that + function walks Instagram's `SNAPSHOT_FIELDS`/`PRESET_FROM_PROFILE` and stamps Instagram's + platform. The mappers have already done the vendor→our-keys work here + (`connectors_tt.normalize_post` / `normalize_comment`), so this adds only what the FETCH knows. + + ⚠ THE TWO WRITE DISCIPLINES ARE OPPOSITE, exactly as on the Instagram side: + * AN IDENTITY ROW OMITS ITS BLANKS, because it is UPSERTED — `upsert_rows` writes the keys it + is handed, so sending "" would let a thin run ERASE what a full one learned. + * A METRIC SNAPSHOT IS APPENDED ONLY WHEN SOMETHING WAS MEASURED. A row in a time series + should mean "this was true then"; a row meaning "nobody looked" eats the cap and draws a + chart of nothing. + + ⛔⛔ D-117 IS NOT REPRODUCED HERE, AND THAT IS THE INSTRUCTION THE TICKET LEADS WITH. + Instagram denormalises `posted_at` and `type` onto its post SNAPSHOT rows, where they never + reconcile against the identity table and quietly become a second, ageing answer to a question + the posts table already answers. This snapshot carries the METRICS and the join key and nothing + else — `TT_PSNAP_METRIC_KEYS` is derived from the declaration minus the keys the snapshot owns, + so the exclusion is structural rather than a list somebody has to remember to keep short. + """ + prof = (res or {}).get("profile") or {} + who = str(prof.get("handle") or "").strip().lstrip("@").lower() + idents, metric_rows, comment_rows = [], [], [] + for p in (res or {}).get("posts") or []: + shortcode = str((p or {}).get("shortcode") or "").strip() + if not shortcode: + continue + ident = {k: v for k, v in p.items() if v not in (None, "")} + # ⚠ The Posts dataset carries `profile_username`, but a row that omits it must still join: + # the profile whose `top_videos` we scraped IS the influencer, and that is a fact of the + # call rather than of the row. + ident.setdefault("influencer_key", who) + ident["platform"] = PLATFORM_TIKTOK + idents.append(ident) + measured = {k: p.get(k) for k in TT_PSNAP_METRIC_KEYS + if k != "source_payload" and p.get(k) is not None} + if not measured: + continue + row = {"platform": PLATFORM_TIKTOK, "shortcode": shortcode, + "influencer_key": str(p.get("influencer_key") or who), + "post_snapshot_key": f"{shortcode}@{pulled}", "pulled_at": pulled} + for k, v in measured.items(): + row[k] = _s(v, 400) + metric_rows.append(row) + for c in (res or {}).get("comments") or []: + if not str((c or {}).get("comment_key") or "").strip(): + continue + row = {k: v for k, v in c.items() if v not in (None, "")} + # Same fact-of-the-call argument: the Comments dataset has no influencer field at all, so + # without this the comments table could never be filtered by creator. + row.setdefault("influencer_key", who) + row["platform"] = PLATFORM_TIKTOK + comment_rows.append(row) + return idents, metric_rows, comment_rows + + +def tt_snapshot_row(res, pulled): + """One TikTok pull → its `ut_tt_snapshots` observation, or None when there is no handle. + + ⛔ THE APPEND LAW, UNCHANGED FROM INSTAGRAM AND FOR THE SAME MEASURED REASON: TikTok's Profiles + dataset carries **no measurement timestamp** (`create_time` is when the ACCOUNT was made, not + when `followers` was true — probed, `tiktok-capture.md`). So the series is dated by when WE + read it, and the key is `@` — idempotent by its KEY rather than by a flag, + so re-running over the same rows rewrites the same row instead of growing the table. + ⚠ `pulled_at` keeps the FULL stamp while the record's `enriched_at` is a day: the record answers + "how stale", the series answers "when exactly", and collapsing the second into the first would + make two reads on one day indistinguishable. + """ + prof = (res or {}).get("profile") or {} + handle = str(prof.get("handle") or "").strip().lstrip("@").lower() + if not handle: + return None + out = {"platform": PLATFORM_TIKTOK, "snapshot_key": f"{handle}@{pulled}", + "influencer_key": handle, "pulled_at": pulled, + "source": _s((res or {}).get("via") or "brightdata", 60)} + for fd in TT_SNAPSHOT_FIELDS: + key = fd["key"] + if key in _TT_SNAPSHOT_OWN_KEYS or key == "platform": + continue + value = prof.get(key) + # ⚠ A ZERO IS A MEASUREMENT AND SURVIVES — the test is `is not None`, never truthiness. + # `str(value or "").strip()` would drop a genuine 0 follower count, and this is a series + # whose whole purpose is that a number moved. + if value is not None and str(value).strip() != "": + out[key] = str(value) if key == "source_payload" else _s(value, 400) + return out + + +def run_field_instagram(rt, defn, username="automation", log=print, step=_no_step, + rows=None): + """Automation #2 (R7): for every row of a database that carries a profile URL, pull the public + profile through the paid capability chain, write a status string into the automation column, + and append a timestamped row to each of the three IG tables. + + ⭐ WAVE 28 / R5 — there is no tier and no rung choice here any more. `pull_profile` routes per + CAPABILITY and reports `blocked` rather than downgrading to an approximate row, so the two + "which rung answered" counters this function used to keep have nothing left to distinguish.""" + cfg = defn.get("config") or {} + table_key, fkey = cfg.get("targetTable"), cfg.get("fieldKey") + post_metrics = bool(cfg.get("postMetrics")) + comment_metrics = bool(cfg.get("commentMetrics")) + dry = bool(cfg.get("dryRun")) + # ⚠ THE STEP KEYS ARE THE CANVAS NODE IDS (contract C3) and the two must move together — a + # status written under a node id `graph()` no longer emits is a dot nothing renders, which is + # indistinguishable from a step that never ran. + steps = {"trigger": "ok", "source": "idle", + "capture_posts": "idle" if post_metrics else "skipped", + "capture_comments": "idle" if comment_metrics else "skipped", + "write": "idle"} + t = ut_get(rt, table_key) + if not t: + steps["source"] = "error" + return ("error", f"{table_key} is not a database in this workspace", {}, [], steps) + url_field = cfg.get("urlField") or _auto_url_field(t, fkey) + if not url_field: + steps["source"] = "error" + return ("error", "the automation column has no URL field bound to it", {}, [], steps) + steps["source"] = "ok" + rows = dict(t.get("rows") or {}) + # the relational tables the pull lands in (R7): every row timestamped for time-range filters + if dry: + # The Write node is off: resolve the keys, create nothing. (`ut_ensure` writes.) + snap_key, post_key, ps_key, comment_key = (IG_SNAPSHOTS_TABLE, IG_POSTS_TABLE, + IG_POST_SNAPSHOTS_TABLE, IG_COMMENTS_TABLE) + else: + graph = ensure_ig_graph(rt, username, str(defn.get("id") or ""), + profile_table=table_key) + snap_key, post_key, ps_key, comment_key = (graph[IG_SNAPSHOTS_TABLE], graph[IG_POSTS_TABLE], + graph[IG_POST_SNAPSHOTS_TABLE], graph[IG_COMMENTS_TABLE]) + missing = [] if dry else ut_missing(rt, snap_key, post_key, ps_key, comment_key) + snaps = dict((ut_get(rt, snap_key) or {}).get("rows") or {}) + posts = dict((ut_get(rt, post_key) or {}).get("rows") or {}) + psnaps = dict((ut_get(rt, ps_key) or {}).get("rows") or {}) + comments = dict((ut_get(rt, comment_key) or {}).get("rows") or {}) + + counts = {"profiles": 0, "ok": 0, "partial": 0, "blocked": 0, "error": 0, + "posts": 0, "new_posts": 0, "paid": 0, "capped": 0, "metrics": 0, "comment_rows": 0, + "missing_tables": len(missing)} + cells, affected, notes = {}, [], [] + pending_metrics = [] + # ⛔ ACCUMULATE HERE, UPSERT ONCE PER TABLE AFTER THE LOOP. This used to call `upsert_rows` + # once per POST, which is O(existing) per call — survivable only while the cap was 5000 rows. + # `MAX_UT_IG_ROWS` makes the same loop hundreds of millions of dict copies, i.e. an automation + # that no longer finishes. **Raising a cap and batching its writer are ONE change.** (W19-C.) + in_snaps, in_posts, in_psnaps, in_comments = [], [], [], [] + targets = [(rid, str(r.get(url_field, "") or "").strip()) + for rid, r in rows.items() if str(r.get(url_field, "") or "").strip()] + for i, (rid, url) in enumerate(targets): + if i: + time.sleep(PACE_SECONDS) # ≥2 s between profiles (R7) + # ITEM 6: the other genuinely long runner — ≥2 s of pacing per profile means a 60-profile + # column automation blocks for minutes by design. A counting step is the difference + # between "working through them" and "stuck". + step(f"Capturing profile {i + 1} of {len(targets)}") + counts["profiles"] += 1 + res = pull_profile(url, max_posts=cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL, log=log, + post_metrics=post_metrics, comment_metrics=comment_metrics, + pending_metrics=(pending_metrics if not dry else None)) + pulled = _iso() + via = res.get("via") or "" + read_ok = res["state"] in ("ok", "partial") + # ⚠ `counts["paid"]` SURVIVES R5 AND IT IS NOT THE RETIRED TIER. It counts profiles a PAID + # vendor actually answered, which is the run's spend report — the thing the owner reads to + # reconcile a bill. What died is the free rung it used to be contrasted with, so the test + # is now simply "did a vendor answer" rather than "did the vendor we were told to try". + if read_ok and via in ("brightdata", "apify"): + counts["paid"] += 1 + if read_ok: + counts["ok" if res["state"] == "ok" else "partial"] += 1 + prof = res["profile"] + snap_row, ident_rows, metric_rows, captured_comments = capture_rows(res, pulled) + in_snaps.append(snap_row) + in_posts.extend(ident_rows) + in_psnaps.extend(metric_rows) + in_comments.extend(captured_comments) + counts["metrics"] += len(metric_rows) + counts["comment_rows"] += len(captured_comments) + counts["posts"] += len(res.get("posts") or []) + detail = f"{prof.get('followers') or '?'} followers" + if prof.get("approx"): + detail += " (approx)" + if res.get("posts"): + detail += f", {len(res['posts'])} posts" + elif res["state"] == "partial": + detail += ", posts not readable on the rung that answered" + cells[rid] = f"{res['state']} · {_stamp()} · {detail}" + elif res["state"] == "blocked": + counts["blocked"] += 1 + cells[rid] = f"blocked · {_stamp()} · {_s(res.get('note'), 90)}" + notes.append(res.get("note") or "blocked") + else: + counts["error"] += 1 + cells[rid] = f"error · {_stamp()} · {_s(res.get('note'), 90)}" + notes.append(res.get("note") or "error") + affected.append(rid) + + # --- THE THREE UPSERTS. Once each, over the whole run's accumulated rows. + snaps, c_snap = upsert_rows(snaps, in_snaps, "snapshot_key", cap=row_cap(snap_key)) + posts, collapsed_posts = dedupe_canonical_rows(posts, "shortcode", newest_by="measured_at") + posts, c_post = upsert_rows(posts, in_posts, "shortcode", cap=row_cap(post_key)) + c_post["duplicates"] += collapsed_posts + psnaps, c_ps = upsert_rows(psnaps, in_psnaps, "post_snapshot_key", cap=row_cap(ps_key)) + comments, collapsed_comments = dedupe_canonical_rows(comments, "comment_key") + comments, c_comments = upsert_rows(comments, in_comments, "comment_key", cap=row_cap(comment_key)) + c_comments["duplicates"] += collapsed_comments + counts["new_posts"] = c_post["inserted"] + capped = [(snap_key, c_snap["capped"]), (post_key, c_post["capped"]), + (ps_key, c_ps["capped"]), (comment_key, c_comments["capped"])] + counts["capped"] = sum(n for _k, n in capped) + + if not dry: + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + tt = cur.get(table_key) + if tt is not None: + for rid, val in cells.items(): + tt.setdefault("rows", {}).setdefault(str(rid), {})[fkey] = val + for k, rws in ((snap_key, snaps), (post_key, posts), (ps_key, psnaps), + (comment_key, comments)): + tgt = cur.get(k) + if tgt is not None: + tgt["rows"] = rws + _refresh_relations_inplace(cur, log=log) + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") # ONE coalesced update for all four tables + queued = queue_pending_metric_snapshots(rt, str(defn.get("id") or ""), pending_metrics) + if queued: + counts["metric_batches_pending"] = queued + + # --- C6 (R2): WRITE-THROUGH to the platform master. Three postures, never conflated: + # `ok` silent, `off` an honest aside (the tenant copy IS the story on this deployment), + # `error` a LOUD partial — a pooled history with silent holes is worse than none. + master_note = "" + if not dry and (in_snaps or in_posts or in_psnaps): + import ig_master + m_status, m_note = ig_master.append_run(getattr(rt, "key", ""), + in_snaps, in_posts, in_psnaps) + if m_status == "error": + master_note = f"⚠ the platform master copy FAILED. {m_note}; the tenant copy " \ + f"is complete and the next run re-appends" + counts["master_failed"] = 1 + elif m_status == "ok": + counts["master"] = len(in_snaps) + len(in_psnaps) + + # ⚠ A RUN IS ONLY 'ok' IF EVERY PROFILE WAS. The first live run rolled up to 'ok' while + # every CELL said `partial`, because the rollup only asked about blocked/error — so the rail + # showed a green dot over a table with no posts in it. A summary that disagrees with the + # cells it summarises is the failure this whole module's honest-status rule exists to + # prevent, so `partial` now propagates. Measured 2026-08-04. + read = counts["ok"] + counts["partial"] + if not read: + state = "error" if counts["error"] else "partial" if counts["blocked"] else "ok" + elif counts["blocked"] or counts["error"] or counts["partial"]: + state = "partial" + else: + state = "ok" + if counts.get("metric_batches_pending"): + state = "partial" + # --- per-node outcomes for the canvas (see the same note in `run_scrape_db`) + if not targets: + cap_state = "idle" + elif not read: + cap_state = "error" if counts["error"] else "blocked" + else: + cap_state = "partial" if (counts["blocked"] or counts["error"] + or counts["partial"]) else "ok" + # ⭐ C3 — THE CAPTURE FORK IS GONE, SO ITS OUTCOME LANDS ON `source`. There is one way to + # read a profile now, and the node that names the profile set is the honest owner of "did + # reading them work". The old `capture`/`capture_paid`/`capture_anon` trio described a branch + # that no longer exists in behaviour OR on screen. + steps["source"] = cap_state if targets else steps["source"] + # ⚠ MEASURED, LIKE EVERY OTHER DOT: it is `ok` only if an engagement row was actually + # appended. "It was switched on" is not the same fact as "it answered", and painting the + # second from the first is the fabrication `node_status` refuses to make. + # ⚠ AND `blocked` IS A CLAIM ABOUT THE VENDOR, so it needs something to have been ASKED. A + # run that found no posts to enrich did not have a rung refuse it — nothing was requested — + # so that reads `skipped`, the same word an off switch earns. + steps["capture_posts"] = ("ok" if counts["metrics"] + else "partial" if counts.get("metric_batches_pending") + else "blocked" if (post_metrics and counts["posts"]) + else "skipped") + steps["capture_comments"] = ("ok" if counts.get("comments") + else "blocked" if (comment_metrics and counts["posts"]) + else "skipped") + + summary = (f"{read}/{counts['profiles']} profiles read, {counts['posts']} posts " + f"({counts['new_posts']} new)") + if counts.get("metric_batches_pending"): + summary += (f", {counts['metric_batches_pending']} post-engagement batch" + f"{'' if counts['metric_batches_pending'] == 1 else 'es'} still building " + "(collected automatically)") + if counts["paid"]: + summary += f", {counts['paid']} with exact counts" + if counts["metrics"]: + summary += f", {counts['metrics']} post engagement snapshots" + elif post_metrics and counts["posts"] and not counts.get("metric_batches_pending"): + # ⚠ ASKED FOR AND NOT DELIVERED IS ITS OWN SENTENCE. Silence here would read as "there + # was no engagement", which is a claim about Instagram rather than about our run. + summary += ", no post engagement was readable" + if counts["comment_rows"]: + summary += f", {counts['comment_rows']} comment rows captured" + elif comment_metrics and counts["posts"]: + summary += ", no comment engagement was readable" + if counts["partial"]: + summary += f", {counts['partial']} profile-only (posts not readable)" + if counts["blocked"]: + summary += f", {counts['blocked']} blocked" + note = cap_note(capped, missing) + if note: + summary += f". {note}" + state = "partial" if state != "error" else state + steps["write"] = "partial" + elif not dry: + steps["write"] = "ok" + if master_note: + summary += f". {master_note}" + state = "partial" if state != "error" else state + steps["write"] = "partial" + if dry: + summary = f"Test run. Nothing saved. Would have written: {summary}" + steps["write"] = "skipped" + state = "partial" if state != "error" else state + if notes: + summary += f". {notes[0][:120]}" + if not dry: + # C7: the fresh master rows are exactly what the table's metric cells summarise. + try: + compute_metric_cells(rt, table_key) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] metric compute {table_key} failed: {type(e).__name__}: {e}") + return (state, summary, counts, affected, steps) + + +def _auto_url_field(table, fkey): + """The URL field an automation column is bound to: its own `automation.urlField`, else the + first url-typed column on the table. Never guessed silently — the caller reports which.""" + for f in table.get("fields") or []: + if f.get("key") == fkey: + bound = ((f.get("automation") or {}).get("urlField") or "").strip() + if bound: + return bound + for f in table.get("fields") or []: + if f.get("type") == "url": + return f.get("key") + return "" + + +#: ⭐ W29-T04 — the discovery-run bookkeeping columns, so `run_discover_tiktok` and the D-116 merge +#: rule agree on what discovery OWNS versus what enrichment owns. Anything not in this set is a +#: MEASUREMENT, and a corpus re-find must never overwrite an enriched measurement with its own +#: cheaper approximation. +TT_DISCOVERY_KEYS = frozenset({"platform", "handle", "created_by", "found_count", + "first_found", "last_found", "source", "source_payload"}) + + +def _tt_candidate_row(row, stamp): + """One TikTok corpus row → a `ut_tt_profile` candidate row. None without a handle. + + ⛔ NEVER EMITS `found_count` — the runner computes it against what is already stored, so + writing it here would reset the counter to 1 on every re-find. Same rule, same reason, as + `_candidate_row` on the Instagram side. + ⭐ THE FIELD MAP IS THE CONNECTOR'S, not a second one written here. `normalize_profile` is what + the enrich path will use too, so a corpus row and a scrape row cannot disagree about which + vendor key becomes which column. + """ + import connectors_tt as _tt + cells = _tt.normalize_profile(row if isinstance(row, dict) else {}) + if not str(cells.get("handle") or "").strip(): + return None + return {**cells, "last_found": stamp, "source": "corpus"} + + + +def _ig_discovery_series(rt, table_key, incoming, stamp, username, auto_id, counts, log): + """⭐⭐ THE FORWARD HALF (2026-08-10), Instagram only. Every candidate a run wrote also leaves + an OBSERVATION behind — see `corpus_snapshot_row`. Without it a backfill is theatre: the next + discovery run re-opens exactly the gap the backfill just closed, which is how a data defect + becomes a recurring one. + + ⚠ `ensure_ig_graph`, NOT a snapshot-only `ut_ensure`, and it is a real behaviour change worth + owning: a tenant that has never enriched gets the four canonical child databases the first time + discovery runs (four of `MAX_UT_TABLES = 40`). The alternative writes a series into a store the + profile table has no LINK to — the numbers would be recorded and unreachable, which is half a + feature wearing a whole one's clothes. + ⚠ `incoming`, not `merged`: the observation belongs to the profiles THIS RUN read, not to every + row the table happens to hold. + """ + try: + graph = ensure_ig_graph(rt, username, auto_id, profile_table=table_key) + counts["series"] = append_ig_snapshots( + rt, [corpus_snapshot_row(c, day=stamp) for c in incoming], + graph[IG_SNAPSHOTS_TABLE], log=log) + except Exception as e: # noqa: BLE001 + # The candidates already landed. A series write that fails must not throw away the rows we + # just paid the vendor for — the same posture the preset top-up takes. + log(f"[aios-auto] discover {auto_id}: corpus series write failed: " + f"{type(e).__name__}: {e}") + + +def _ig_discovery_metrics(rt, table_key, auto_id, log): + """C7 — the metric FIELDS computed off the master series. See the field runner.""" + try: + compute_metric_cells(rt, table_key) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] metric compute {table_key} failed: {type(e).__name__}: {e}") + + +#: ⭐⭐ WAVE 30 · T09 (DEBT D-129) — ONE DISCOVERY RUNNER, TWO CORPORA, AND EVERY DIFFERENCE BETWEEN +#: THEM IS A ROW IN THIS TABLE. +#: +#: `run_discover_tiktok` was written in wave 29 as a deliberate SIBLING of `run_discover_instagram` +#: — its own docstring said so and gave the reason: *"it would rewrite the one code path this +#: product bills money through"*. That was the right call with one platform's worth of evidence and +#: it stopped being right when the duplication reached ~210 lines, because a sibling does not +#: inherit fixes. The proof is already in the copy below: the TikTok runner carries the **D-116** +#: precedence rule (a corpus re-find must not overwrite an ENRICHED measurement with the corpus's +#: cheaper approximation) and the Instagram one, which is the one with paying rows in it, does not. +#: +#: ⛔ **THE COLLAPSE IS BEHAVIOUR-PRESERVING, DELIBERATELY, AND `discovery_keys` IS WHERE YOU CAN +#: SEE THAT DECISION RATHER THAN INFER IT.** Setting Instagram's to `IG_DISCOVERY_KEYS` would close +#: D-116 in one line — and it would change what a live, billed, nightly automation writes to rows +#: the owner reads, inside a refactor whose whole claim is that nothing moved. It is REPORTED +#: instead (see D-116's row), which is the standing rule for a limit that is not being removed +#: today. ⚠ The one-line fix is now visible in a table instead of buried 300 lines apart, which is +#: most of the value of doing this at all. +#: +#: ⚠ `dataset` IS A CALLABLE, not a string: `connectors_tt` imports this module at import time, so +#: the engine may only reach it from INSIDE a function. `mapper` has the same shape for the same +#: reason on the TikTok side. +#: ⚠ `handle_field` is the column the engine-added `not_in` exclusion names, and it is per platform +#: because B-20 MEASURED that Bright Data's TikTok Profiles dataset calls Instagram's `account` +#: `account_id`. The exclusion never appears as a condition row, so nothing a person types can +#: correct it. +DISCOVERY_SPECS = { + "discover_instagram": { + "noun": "profile", + "log_tag": "discover", + "platform": PLATFORM_INSTAGRAM, + "dataset": lambda: BD_DS_PROFILES, + "handle_field": "account", + "mapper": lambda row, stamp: _candidate_row(row, stamp), + "table": DISCOVER_TABLE, + "label": DISCOVER_LABEL, + "fields": CANDIDATE_FIELDS, + # ⛔ D-116 IS OPEN ON THIS SIDE. `None` = discovery overwrites whatever it re-finds. + "discovery_keys": None, + "on_write": _ig_discovery_series, + "after_write": _ig_discovery_metrics, + }, + "discover_tiktok": { + "noun": "TikTok profile", + "log_tag": "discover-tt", + "platform": PLATFORM_TIKTOK, + "dataset": lambda: _tt_module().TT_DS_PROFILES, + "handle_field": "account_id", + "mapper": lambda row, stamp: _tt_candidate_row(row, stamp), + "table": TT_PROFILE_TABLE, + "label": TT_TABLE_LABELS[TT_PROFILE_TABLE], + "fields": TT_PROFILE_FIELDS, + "discovery_keys": TT_DISCOVERY_KEYS, + # ⚠ NO series and NO metric fields on this side, and that is not an oversight: both hooks + # write into Instagram's own snapshot store, through Instagram's own vocabulary. TikTok's + # series is W29-T06's `ut_tt_post_snapshots`, which the ENRICH path owns. + "on_write": None, + "after_write": None, + }, +} + + +def _tt_module(): + """`connectors_tt`, imported LAZILY. It imports this module at module level, so a top-level + import here would close the cycle.""" + import connectors_tt as _tt + return _tt + + +def run_discovery(rt, defn, username="automation", log=print, step=_no_step, rows=None): + """Automation #3 (owner ruling R7 / D-23) and DEBT D-9: FIND handles nobody here has typed in, + on whichever corpus this automation's KIND names. + + Two-phase by construction — see the DISCOVERY section header. A run either STARTS a corpus + query and hands its snapshot id to the next run, or COLLECTS one a previous run started. Every + outcome is a sentence about what actually happened; none of them is a green zero. + + ⛔ THE PLATFORM COMES FROM `defn["kind"]`, WHICH IS THE ONLY THING THAT DECIDES IT NOW. Before + wave 30 the choice was made by WHICH FUNCTION `RUNNERS` pointed at, so a fixture (or a stored + definition) with a stale kind still ran the corpus its caller had in mind. It cannot any more: + an unknown kind is refused rather than defaulted, because defaulting here means billing one + platform's corpus for another platform's search. + """ + kind = str(defn.get("kind") or "") + spec = DISCOVERY_SPECS.get(kind) + cfg = defn.get("config") or {} + dry = bool(cfg.get("dryRun")) + limit = int(cfg.get("recordsLimit") or 5) + preds = list(cfg.get("predicates") or []) + auto_id = str(defn.get("id") or "") + steps = {"trigger": "ok", "find": "idle", "collect": "idle", "write": "idle"} + counts = {"asked": limit, "found": 0, "new": 0, "seen_again": 0, "capped": 0, + "missing_tables": 0} + if spec is None: + # ⛔ NEVER A DEFAULT. `RUNNERS` maps exactly the kinds in `DISCOVERY_SPECS` onto this + # function, so reaching here means somebody widened one of the two without the other. + steps["find"] = "blocked" + return ("error", f"this automation's kind ({kind or 'blank'}) has no profile corpus, so " + f"nothing was searched and nothing was charged", counts, [], steps) + noun = spec["noun"] + + if not bd_ready(): + # ⛔ THE FAIL-CLOSED PATH (C4). Discovery has NO anonymous rung to drop to — the corpus is + # the vendor's — so this is where the run stops, saying which env var is missing. + steps["find"] = "blocked" + return ("partial", "Profile search is not set up yet. Nothing was searched and nothing " + "was charged", counts, [], steps) + + pending = str((defn.get("state") or {}).get("pendingSnapshot") or "") + if not pending: + # C4 (wave 22): the guard holds at RUN too — a stored config can predate the law, and + # the vendor bills for breadth whether the filter was saved yesterday or last month. + # D-68: the SHAPE guard rides beside the breadth guard, at save AND at run - a config + # stored before this check existed must not reach the vendor either. + # ⭐ W32-T46: the RUN-time twin of the save-time guard says the right network's name too — + # `defn["kind"]` is what decides the corpus everywhere else in this function. + guard_err = (narrowing_refusal(preds, cfg.get("operator"), str(defn.get("kind") or "")) + or depth_refusal(preds, cfg.get("operator"))) + if guard_err: + steps["find"] = "blocked" + return ("error", f"the search was not started. {guard_err}", counts, [], steps) + if dry: + est = discover_estimate(limit) + steps["find"] = steps["write"] = "skipped" + return ("partial", + f"Test run. Nothing saved. Would look for up to {limit} " + f"{noun}{'' if limit == 1 else 's'} matching " + f"{_predicate_sentence(preds, cfg.get('operator'))}, for about " + f"${est['usd']}", counts, [], steps) + step("Starting the search") + # ⭐ ITEM 7 — DO NOT PAY FOR WHAT WE ALREADY HAVE. The exclusion is engine-added: it never + # appears as a condition row, and `bd_filter_start` drops it rather than risk a shape the + # vendor refuses. Reported either way, because "why did this run find fewer" and "why did + # this run cost the same as last night" are both questions this line answers. + # ⚠ `already_found_handles` NEEDS NO PER-PLATFORM BRANCH — it is scoped to THIS definition's + # own tables (`automation_tables`), so a TikTok automation targeting `ut_tt_profile` + # excludes exactly the handles it has itself found. Verified rather than assumed: the + # function reads `config.targetTable` plus each `create_record` action's table, and names + # no IG constant. + excl = {} + sid, note = bd_filter_start(preds, cfg.get("operator") or "and", limit, + dataset_id=spec["dataset"](), + exclude_handles=already_found_handles(rt, defn), + applied=excl, handle_field=spec["handle_field"]) + if excl.get("excluded"): + step(f"excluded {excl['excluded']} already-found " + f"profile{'' if excl['excluded'] == 1 else 's'} at the provider") + elif excl.get("dropped"): + step(f"searching without the already-found list. {excl['dropped']}") + if note: + steps["find"] = "blocked" + return ("error", f"the search was not started. {note}", counts, [], steps) + pending = sid + # ⛔ LOG THE ID *BEFORE* PERSISTING IT, and the order is the whole point. `set_state` goes + # through `rt.update`, which can RAISE on an unavailable store — and that exception + # becomes an error run, so a `log()` placed after it never executes. The snapshot would + # then exist, be billed, and have its id in neither the store nor the logs. Logging first + # means the worst case is still recoverable by a human reading the Space output. + log(f"[aios-auto] {spec['log_tag']} {auto_id}: started {sid}") + # PERSIST BEFORE POLLING. A snapshot the vendor is already building does not stop + # existing because this process dies thirty seconds later, and a lost id is a set we + # paid attention to and can never collect. + set_state(rt, auto_id, {"pendingSnapshot": sid, "pendingSince": _iso()}) + steps["find"] = "ok" + + waited, status, size, note = 0.0, "", 0, "" + while True: + # ⭐ ITEM 6 — THE ONE PLACE THE LIVE STEP HAS TO MOVE. This loop blocks for up to + # BD_FILTER_WAIT at the vendor, and D measured that this wait IS the whole of "Run once now + # is laggy / looks stuck". A counter here is what makes a legitimate wait distinguishable + # from a hung thread; without it the rendered step reads the same word for both, which is a + # progress indicator that cannot indicate progress. + step(f"Searching. {int(waited)}s of " + f"{int(BD_FILTER_WAIT)}s") + status, size, note = bd_filter_status(pending) + if note: + steps["collect"] = "error" + set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None}) + return ("error", note, counts, [], steps) + if status != "building" or waited >= BD_FILTER_WAIT: + break + time.sleep(BD_FILTER_POLL) + waited += BD_FILTER_POLL + step("Reading the profiles the search found") + + if status == "building": + # ⚠ THE HANDOFF, AND IT IS A SUCCESSFUL OUTCOME OF A SORT. Measured build latency is ~20 + # minutes; holding a worker thread that long on a free-tier container to poll would be + # the wrong shape. Say plainly that it is still running and who collects it. + # ⚠ THE SENTENCE NAMES WHO COLLECTS IT, because the old one did not and that is what made + # this read as "stuck": *"The next run picks up the results"* is a promise about a run + # that, on a MANUAL automation, nobody had scheduled. `pending_collect_ids` makes the tick + # finish it, so the promise is now kept by something rather than by the reader. + # ⭐ WAVE 27 ITEM 14 (owner) — THE BRAG IS GONE. The sentence used to explain the wait by + # naming the corpus size. It was TRUE, and it was answering a question the person had not + # asked. It was also the one place a vendor's catalogue size was quoted to a customer, so + # it would have gone stale the day the vendor grew. THE SHAPE OF THE ANSWER SURVIVES and is + # the part that mattered: the wait does not shrink when you ask for fewer records, and a + # person who does not know that reads "20 minutes for 10 rows" as a fault. + mins = 0 + try: + since = (defn.get("state") or {}).get("pendingSince") + if since: + mins = max(0, int((_dt.datetime.now(_dt.timezone.utc) + - _dt.datetime.fromisoformat(str(since))).total_seconds() + // 60)) + except Exception: # noqa: BLE001 + mins = 0 + been = f". {mins} min so far" if mins else "" + steps["collect"] = "partial" + return ("partial", + f"Still searching at the provider{been}. A corpus search takes about 20 minutes " + f"however few records you asked for. Nothing is lost, you are not charged twice, " + f"and the results are collected automatically as soon as they are ready", + counts, [], steps) + if status == "empty": + # A search that matched nothing RAN CORRECTLY. It is the ordinary result of a keyword + # that is too specific, so it says what to do about it instead of reporting a fault. + steps["collect"] = "ok" + steps["write"] = "skipped" + set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None}) + return ("partial", "No profiles matched. Try a shorter or more common keyword. " + "'floral' finds more accounts than 'floral design studio'", + counts, [], steps) + if status != "ready": + steps["collect"] = "error" + set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None}) + return ("error", f"the search ended as {status or 'unreadable'} and returned nothing", + counts, [], steps) + + rows, dnote = bd_filter_rows(pending) + if dnote: + # Delivery lags `ready` by minutes (measured). Keep the id; the next run collects it. + steps["collect"] = "partial" + return ("partial", f"the search finished with {size} match" + f"{'' if size == 1 else 'es'} but {dnote}", counts, [], steps) + set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None}) + steps["collect"] = "ok" + + stamp = _iso() + _map = spec["mapper"] + incoming = [c for c in (_map(r, stamp) for r in rows) if c] + counts["found"] = len(incoming) + # ⚠ COUNT THE ROWS WE COULD NOT USE. A corpus row with no handle cannot be a candidate, and + # dropping it silently means "the vendor delivered 50 and we kept 3" reads identically to + # "the vendor delivered 3". It also catches the shape where an error document comes back + # down the rows path and parses as one unusable row instead of an error. + counts["dropped"] = max(0, len(rows) - len(incoming)) + label = cfg.get("targetLabel") or spec["label"] + table_key = (ut_key_for(label, cfg.get("targetTable") or spec["table"]) if dry + else ut_ensure(rt, label, spec["fields"], username, + key=cfg.get("targetTable") or spec["table"], + flow_tag=auto_id, lock_fields=True)) + existing = dict((ut_get(rt, table_key) or {}).get("rows") or {}) + # --- ⭐ WAVE 26 · C3 / owner ruling R4 — THE UPSERT KEY IS `(platform, handle)`. + # + # ⛔ THIS RETIRES WAVE 22's C6 COMPOUND `(handle, created_by)`, deliberately, and the reasoning + # that built it is worth stating before it is discarded rather than after: per-user rows gave + # each person their own `found_count` and their own review card, so two colleagues scouting the + # same market never edited each other's work. Owner, 2026-08-06: one profile is ONE row. The + # tenant is the unit, not the user. `created_by` survives as "Found by" — informational, and no + # longer part of the identity. + # + # ⚠ AND THE OWNER ADDED THE GUARD THAT MAKES IT SAFE, WHICH IS THE HALF THAT WOULD HAVE BEEN + # MISSED: a handle is only unique WITHIN a network. `@inayma` on Instagram and `@inayma` on + # TikTok are routinely different people, so keying on the handle ALONE would have silently + # merged two accounts into one row the day the TikTok runner ships (D-9) — a data-loss bug with + # no error, discovered months later by someone wondering why a creator's followers halved. + # Hence `platform`, and hence it being part of the key rather than a label beside it. + _ck = candidate_key + + # ⚠ A ROW WITH NO `platform` CELL IS STAMPED WITH **THIS CORPUS'S** PLATFORM, and the default is + # per spec rather than global. On the Instagram table every legacy row came from the Instagram + # runner, so `Instagram` is a FACT about the stored data; in a `ut_tt_*` table the same blank + # means TikTok for exactly the same reason. Blank-keyed rows would fail to match their own + # re-find and duplicate the whole table on the next run. + for r in existing.values(): + if not str(r.get("platform") or "").strip(): + r["platform"] = spec["platform"] + existing2 = {rid: {**r, "_ckey": _ck(r.get("platform"), r.get("handle"))} + for rid, r in existing.items()} + # `found_count` is ARITHMETIC OVER WHAT IS STORED, not a value from the vendor — re-finding + # a profile is the signal that it keeps matching, so the counter grows instead of + # resetting. `first_found` is written only when the pair is new, so it never moves. + seen = {r["_ckey"]: r for r in existing2.values()} + _dkeys = spec["discovery_keys"] + for c in incoming: + # "Found by" — the first finder is recorded and later finders do not overwrite them, which + # is what the column means now that it is no longer part of the identity. + c["created_by"] = username + c["_ckey"] = _ck(c.get("platform"), c["handle"]) + prev_row = seen.get(c["_ckey"]) + if prev_row and str(prev_row.get("created_by") or "").strip(): + c["created_by"] = prev_row["created_by"] + if prev_row: + counts["seen_again"] += 1 + c["found_count"] = str((_ig_int(prev_row.get("found_count")) or 0) + 1) + # ⛔⛔ DEBT D-116 — THE ENRICHED-MEASUREMENT PRECEDENCE RULE, AND IT RUNS ONLY WHERE THE + # SPEC DECLARES A `discovery_keys` SET. A discovery re-find OVERWRITES an enriched + # `followers` with the corpus number — the corpus is cheaper, rounder and older than a + # scrape, so the row silently gets WORSE every night while the automation reports + # success. The fix is a precedence rule, not a blank check: a row that has been ENRICHED + # (`enriched_at` is stamped only by the enrich path) keeps its exact measurements, and + # discovery may only fill what is genuinely empty and update its own bookkeeping. + # ⚠ The declared set is what discovery OWNS; everything else on a corpus row is a + # measurement, and an approximation must never replace an exact one. + if _dkeys and str(prev_row.get("enriched_at") or "").strip(): + for k in list(c): + if (k not in _dkeys and k != "_ckey" + and str(prev_row.get(k) or "").strip()): + c.pop(k) + else: + counts["new"] += 1 + c["found_count"] = "1" + c["first_found"] = stamp + # ⭐ WAVE 26 · ITEM 1 — THE STAGE STAMP IS PER-AUTOMATION, SO IT CANNOT HANG OFF + # "is this row new to the TABLE". + # + # ⛔ MEASURED DEFECT, two discovery automations pointed at one database: the second one's + # board was EMPTY. It reported "2 profiles found — 0 new, 2 seen before", which is true, + # and then drew nothing, because this stamp lived inside the `else` above. `skey` is + # `stage_`; the branch it sat in asks whether some OTHER automation had + # already created the row. So the first automation to reach a handle claimed it, and every + # later automation sharing that database silently had no cards — with a summary saying it + # had found them. + # + # ⚠ THE ORIGINAL LAW IS PRESERVED, and it is the reason this is a condition rather than an + # unconditional write: a re-find must never pull a card somebody has already moved back + # into Review. So the question is "has THIS automation placed this record yet?" — not "is + # this record new?" A blank stage cell for this automation means unplaced, which is + # exactly the state a new card is in. + merged, mc = upsert_rows(existing2, incoming, "_ckey", cap=row_cap(table_key)) + merged = {rid: {k: v for k, v in r.items() if k != "_ckey"} + for rid, r in merged.items()} + counts["capped"] = mc["capped"] + missing = [] if dry else ut_missing(rt, table_key) + counts["missing_tables"] = len(missing) + if not dry and not missing: + ut_write_rows(rt, table_key, merged) + steps["write"] = "ok" + if spec["on_write"]: + spec["on_write"](rt, table_key, incoming, stamp, username, auto_id, counts, log) + elif dry: + steps["write"] = "skipped" + + affected = [rid for rid, r in merged.items() + if str(r.get("handle")) in {c["handle"] for c in incoming}][:200] + est = discover_estimate(counts["found"]) + summary = (f"{counts['found']} {noun}{'' if counts['found'] == 1 else 's'} found. " + f"{counts['new']} new, {counts['seen_again']} seen before. About " + f"${est['usd']}") + if counts["dropped"]: + summary += (f". {counts['dropped']} result{'' if counts['dropped'] == 1 else 's'} had no " + f"username and could not be saved") + cnote = cap_note([(table_key, counts["capped"])], missing) + if cnote: + summary += f". {cnote}" + steps["write"] = "partial" + if dry: + summary = f"Test run. Nothing saved. Would have written: {summary}" + state = "partial" if (dry or counts["capped"] or missing or not counts["found"]) else "ok" + if not dry and not missing and spec["after_write"]: + spec["after_write"](rt, table_key, auto_id, log) + log(f"[aios-auto] {spec['log_tag']} {auto_id} -> {table_key}: {summary}") + return (state, summary, counts, affected, steps) + + +def _predicate_sentence(preds, operator="and"): + """A predicate list as something a person can read back. Used in summaries and the canvas. + + Human words on BOTH halves — it used to render `biography includes floral`, the vendor's + column name beside the vendor's operator token, in a sentence shown to a customer. + """ + joiner = " or " if str(operator or "").lower() == "or" else " and " + parts = [] + for p in preds or []: + v = p.get("value") + shown = " or ".join(str(x) for x in v) if isinstance(v, list) else v + parts.append(f"{field_label(p.get('name'))} " + f"{BD_OP_LABELS.get(p.get('operator'), p.get('operator'))}" + + ("" if shown is None else f" {shown}")) + return joiner.join(parts) or "no conditions" + + +#: How many records ONE `plain` run walks. Deliberately the ut table's OWN row ceiling rather +#: than a second, smaller number nobody could explain: a plain automation's records ARE its +#: table's rows. What this actually bounds is the APPEND tables (`MAX_UT_IG_ROWS`, 200k), which a +#: flow could be pointed at and which would not finish. +#: ⚠ W31-T36 (D-143 / R6) — THIS IS NOW THE FALLBACK ONLY: what a flow may walk when the table's +#: own limit cannot be resolved (no runtime, or a key `core.user_tables` does not know). The real +#: answer is per TABLE and comes from `core.user_tables.row_limit`; see `_flow_record_cap`. +#: ⛔ Do not re-point a caller at this constant — a private copy of "is this database bounded" is +#: exactly the second evaluator D-143 was. +MAX_FLOW_RECORDS = MAX_UT_ROWS + + +def _flow_record_cap(table_key, rt=None): + """How many records may a flow walk in this database? `None` means NO CAP (R6). + + ⛔ DELEGATES, NEVER DECIDES. `core.user_tables.row_limit` is the one evaluator for "is this + database bounded", and it distinguishes three states this module must not re-derive: `0` + (read-through — its rows are not in the document at all), `None` (connected and UNCAPPED, which + is R6's whole point), and `MAX_ROWS` (the editable substrate, genuinely bounded by the shared + document). A second copy here is exactly the defect D-143 IS. + + ⚠ `0` MEANS "NO ROWS LIVE HERE", NOT "WALK NOTHING". A read-through grid's rows arrive through + the mirror route, so a flow that reached this function already holds ids from somewhere else; + capping it at zero would refuse a walk whose records are in hand. Treated as uncapped, and the + honest place to fix a read-through walk is the walker. + """ + try: + import core.user_tables as _ut_cap + cap = _ut_cap.row_limit(str(table_key or ""), st=rt) + except Exception: # noqa: BLE001 + return MAX_FLOW_RECORDS # the fallback, and it is the conservative direction + return None if not cap else int(cap) + + +def _flow_cap_reason(table_key, rt=None): + """R6's SECOND sentence, appended to the run's own note: the cause and the recommended fix. + + ⛔ R6 IS TWO SENTENCES AND THE SECOND IS THE ONE THAT GETS DROPPED — *"if there is lag or it + can't be done, you need to explicitly tell me why and recommend a fix."* A run that says only + *"walked the first 5000"* has disclosed the number and hidden everything a person could act on. + `core.user_tables.limit_report` already carries both, so this READS them rather than writing a + second wording that would drift from the one the grid shows. + """ + try: + import core.user_tables as _ut_cap + rep = _ut_cap.limit_report(str(table_key or ""), st=rt) or {} + except Exception: # noqa: BLE001 + return "" + cause, fix = str(rep.get("cause") or ""), str(rep.get("recommendation") or "") + if not cause and not fix: + return "" + return f". {cause}." + (f" To walk them all: {fix}." if fix else "") + + +# --------------------------------------------------------------------------------------------- +# WAVE 35 · T36 / CONTRACT C8 / OWNER RULING R10 — THE STATEMENTS BATCH: ASSEMBLE, PARK, NEVER SEND +# +# ⛔⛔ THE TICKET'S `how:` SAID TO USE "the board's strict review posture (wave 22)". THAT POSTURE +# DOES NOT EXIST, and rebuilding it would REVERSE AN OWNER RULING rather than merely miss a symbol. +# Wave 27 R3 deleted `board()`, `move_card()`, `stages_for()`, `ensure_stage_field()` and the review +# branch of the action walk (the tombstone is above `LANE_OPS`), and its reason is a DATA reason, +# verbatim: *"the board wrote MACHINE COLUMNS INTO A TENANT'S OWN TABLE — a stage select, an `_at` +# stamp and a `_cycles` counter per automation — to render a view of state the RUN LOG already +# holds."* `retire_automation_stage_fields` and `migrate_ig_tables` still DROP those columns, so a +# batch parked in a stage field would be deleted by our own migration. +# ⇒ The batch parks on the AUTOMATION DEFINITION, the shape `runs` and `reviews` already use. Zero +# machine columns on a customer's database, and the migration has nothing to take away. +# +# ⛔ A STATEMENTS FLOW IS **BATCH**-SCOPED, NOT RECORD-SCOPED, and that is why it does not go +# through `apply_actions` at all. `run_plain` walks every row of the flow's bound table, so a +# per-record arm would assemble the whole worklist once per record; and a statements agent binds NO +# table (its customers come from the Odoo AR worklist, not a `ut_*` grid), so `run_plain` would +# have answered `partial: "no database is bound yet"` and the step would NEVER HAVE RUN. The +# precedent for a kind that defines its own scope is already here: the enrich-only branch below, +# whose saved View "ARE the set of records the automation was asked to process". +# ⚠ The `_walk` arm for this kind therefore stays a REFUSAL, not a duplicate: a `send_statement` +# dropped into an ordinary record-walking flow must say it did nothing, never half-send. +# +# ⛔⛔ AND NOTHING HERE SENDS. Not one function in this section imports the mail path. The send door +# is `routes_statements`, behind `admin_gate` + `_royal_only`, calling `collections_send. +# queue_statement`, whose SAFE_MODE guardrail lives in the DATA LAYER where no route, payload or UI +# can bypass it. This module ASSEMBLES and PARKS. A person clicks Send. + +#: How many statements one parked batch holds. Bounded for the reason `MAX_RUNS` and `MAX_REVIEWS` +#: are: this rides inside the automation DEFINITION, and an unbounded list is a serialisation cost +#: on every read of the automations bucket. +#: ⚠ WHEN IT BITES IT IS DISCLOSED, never a silent `[:N]` — R6's second sentence, and +#: [[no-unverifiable-aggregates]]. `assemble_statements` appends a note naming the number left out. +MAX_STATEMENT_BATCH = 200 + +#: The collection worklist loader, injectable so a gate can drive this without Odoo credentials. +#: Production leaves it None and the real data layer answers. Same shape as `_DRAFT_CHAT`. +_STATEMENT_ROWS = [None] + + +def _collections(): + """`modules.collections_send`, imported lazily — `platform/` is not on the path at import time + for every consumer of this module, and this is the only section that needs it.""" + import modules.collections_send as cs # noqa: PLC0415 + return cs + + +def statement_steps(defn): + """The ENABLED `send_statement` actions of this flow, top level only. + + ⚠ Top level only, deliberately, and `is_statement_flow` is what makes that safe: a batch flow + is exactly one enabled action. A `send_statement` nested inside an If is NOT a batch flow, so + it falls to the ordinary record walk and is refused there with a sentence. + """ + return [a for a in (((defn or {}).get("flow") or {}).get("actions") or []) + if isinstance(a, dict) and a.get("kind") == "send_statement" + and a.get("enabled", True)] + + +def is_statement_flow(defn): + """Is this automation a statements BATCH rather than a record walk? + + ⛔ EXACTLY ONE ENABLED ACTION, AND IT IS THIS KIND. The narrowness is the safety: a flow that + also updates records has record semantics that the batch path would silently drop, so it keeps + the ordinary walk (where the arm refuses and says so). This mirrors the enrich-only branch in + `run_plain`, which is narrow for the same stated reason. + """ + enabled = [a for a in (((defn or {}).get("flow") or {}).get("actions") or []) + if isinstance(a, dict) and a.get("enabled", True)] + return len(enabled) == 1 and enabled[0].get("kind") == "send_statement" + + +def assemble_statements(defn, log=print): + """`(batch, notes)` — the statements this configuration WOULD send, rendered, plus why anybody + was left out. **Reads Odoo read-only. Sends nothing. Writes nothing.** + + Each entry is `{customer, to, subject, html, tier, overdue}`. `html` is rendered by the SAME + `render_statement_html` the send door uses, so what a person approves is what goes out — a + review screen rendering its own approximation of the mail is a review of the wrong thing. + + ⚠ A CUSTOMER WITH NO EMAIL IS *SKIPPED AND NAMED*, never dropped. `routes_statements.send` + already separates "we could not" from "there was nowhere to send"; this keeps that distinction + at the assembly end, because a batch that silently shrinks is one nobody can reconcile. + """ + cfg = (statement_steps(defn) or [{}])[0].get("config") or {} + cs = _collections() + notes = [] + loader = _STATEMENT_ROWS[0] + rows = list(loader() if loader else cs.load_collection_list(cs.Odoo())) + tier = str(cfg.get("tier") or "").strip() + if tier: + before = len(rows) + rows = [r for r in rows if str(r.get("Tier") or "") == tier] + log(f"[aios-auto] statements: {len(rows)} of {before} customers are in tier {tier}") + subject_tpl = str(cfg.get("subject") or "") or cs.DEFAULT_SUBJECT + intro_tpl = str(cfg.get("intro") or "") or cs.DEFAULT_INTRO + footer_tpl = str(cfg.get("footer") or "") or cs.DEFAULT_FOOTER + month = _dt.date.today().strftime("%B %Y") + batch, no_email = [], [] + for row in rows: + if len(batch) >= MAX_STATEMENT_BATCH: + break + name = str(row.get("Customer") or "") + to = str(row.get("Email") or "").strip() + if not to: + no_email.append(name) + continue + try: + subject = subject_tpl.format(customer=name, company=cs.COMPANY, month=month) + except (KeyError, IndexError): + # An unknown placeholder is the person's typo, not a crash. Show what they typed — + # the same choice `routes_statements.preview` already makes. + subject = subject_tpl + batch.append({"customer": name, "to": to, "subject": _s(subject, 200), + "html": _s(cs.render_statement_html(row, intro_tpl, footer_tpl), 40000), + "tier": str(row.get("Tier") or ""), "overdue": row.get("Overdue")}) + if no_email: + shown = ", ".join(no_email[:5]) + one = len(no_email) == 1 + notes.append(f"{len(no_email)} customer{'' if one else 's'} " + f"{'has' if one else 'have'} no email address on their record and " + f"{'is' if one else 'are'} not in this batch " + f"({shown}{', and others' if len(no_email) > 5 else ''}). " + f"Add an address in Odoo and run it again") + left = len(rows) - len(batch) - len(no_email) + if left > 0: + # R6's second sentence: a limit that bites is REPORTED, with its cause and what to do. + notes.append(f"{left} more customers matched but one batch holds " + f"{MAX_STATEMENT_BATCH}. Send this batch, then run it again for the rest, or " + f"narrow the tier filter") + return batch, notes + + +def park_statements(rt, auto_id, batch, notes): + """Write the assembled batch onto the DEFINITION as `pendingStatements`. Replaces, never + appends: a batch is THIS run's worklist, and two runs' statements merged into one list is a + customer invoiced twice. + + ⚠ `flush="async"` for the reason every writer in this module gives — the read-your-writes + contract means the next `GET /automations` already sees it, and only the upload is deferred. + """ + entry = {"ts": _iso(), "count": len(batch), "sent": False, + "items": list(batch)[:MAX_STATEMENT_BATCH], + "notes": [_s(n, 300) for n in (notes or [])][:25]} + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + d = cur.get(str(auto_id)) + if d is not None: + d["pendingStatements"] = entry + return cur + + _store_update(rt, _up, flush="async") + return entry + + +def clear_statements(rt, auto_id): + """Drop a parked batch — after it is sent, or when somebody discards it.""" + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + d = cur.get(str(auto_id)) + if d is not None: + d.pop("pendingStatements", None) + return cur + + _store_update(rt, _up, flush="async") + + +def run_statements(rt, defn, username="automation", log=print, step=_no_step): + """The batch runner: assemble, park, tell somebody. Returns `run_plain`'s 5-tuple. + + ⛔ THE STATE IS `partial`, NEVER `ok`, AND THAT IS DELIBERATE. `ok` would put a green dot over + a job that is only half done — the statements exist and NOBODY HAS SENT THEM. `partial` is this + module's honest-progress state and the summary says exactly what is waiting, which is what makes + the run notification (`_notify_run`) read as "come and look" rather than "done". + """ + if not is_statement_tenant(rt): + # Belt and braces behind `clean_actions`' wall: a definition stored before the wall, or + # copied between tenants, must not assemble another company's customer list. + return ("error", "statements are not configured for this workspace", {}, [], {}) + step("reading the collection list") + try: + batch, notes = assemble_statements(defn, log=log) + except Exception as exc: # noqa: BLE001 + return ("error", f"the collection list could not be read: {str(exc)[:200]}", {}, [], {}) + step("rendering statements") + park_statements(rt, defn.get("id"), batch, notes) + counts = {"statements": len(batch)} + if not batch: + return ("partial", "no customers matched, so there is nothing to review", counts, [], {}) + one = len(batch) == 1 + return ("partial", + f"{len(batch)} statement{'' if one else 's'} " + f"{'is' if one else 'are'} ready for review. Nothing has been sent. Open this agent " + f"and click Send to release them", + counts, [], {}) + + +#: ⭐⭐ WAVE 35 · T37 — THE SEEDED ROYAL IMPORTS STATEMENTS AGENT. +#: A FIXED id, because the row's own EXISTENCE is the idempotency key (see `seed_statements_agent`). +STATEMENTS_AGENT_ID = "system:statements" +SYSTEM_STATEMENTS = "statements" +#: First of the month, 06:00. A monthly statement run is what the collections worklist is for, and +#: the hour matches the Odoo sync default so the numbers it reads are the night's. +STATEMENTS_DEFAULT_CRON = "0 6 1 * *" + + +def seed_statements_agent(rt, username="system"): + """Mint Royal Imports' "Monthly statements" agent ONCE, **switched OFF**. Returns the + definition if it wrote one, else None. + + ⛔⛔ IT ARRIVES `enabled: False` AND THAT IS THE TICKET'S OWN TRAP, not a preference. An agent + that ships enabled has scheduled itself against real customers before a single person has read + its configuration — and this one's action queues mail to the tenant's actual debtors. + + ⭐ WHY STORED, WHEN WAVE 34's SYSTEM AGENT IS DERIVED. `_odoo_sync_row` gives three reasons for + deriving and only one survives contact with THIS agent (raised as `ASK D-11`): + · "two copies of one cadence" — does not apply: Odoo's cadence already lives in the connector + config, so a stored copy could drift; a statements schedule has no other home to drift from. + · "stored means the failure-pause can disable it" — INVERTS: auto-pausing statements after K + consecutive failures is correct (a dead credential should stop it), where pausing the Odoo + cadence would silently break infrastructure. + · "nothing can seed it" — real, and answered here: ONE write per tenant ever, by the + CONTAINER (which is what D-195 asks for; what it forbids is a CLI writing while the Space + is up), behind the existence check below. + And deriving cannot meet T37 anyway: a derived row is not in the automations bucket, so edit, + toggle, schedule and run would each need a bespoke door — four new mechanisms to avoid one + guarded write. The Odoo agent escapes that only because a separate resync loop already does its + work. + + ⛔ THE EXISTENCE CHECK IS THE WHOLE IDEMPOTENCY STORY, AND IT IS SAFE ONLY BECAUSE THE ROW + CANNOT BE DELETED. `routes_automation.delete_automation` refuses any stored row carrying + `system` (409, with a sentence), so "the row is present" can never become false behind our + back. If that refusal is ever relaxed, THIS FUNCTION NEEDS A SEPARATE DURABLE FLAG — otherwise + deleting the agent resurrects it on the next list call, which is a delete that undoes itself. + ⚠ And `system` is STICKY through `clean_definition` (see its return) for the same reason: an + edit that stripped the marker would make the row deletable and re-open exactly that hole. + """ + if not is_statement_tenant(rt): + return None + existing = all_definitions(rt) or {} + if STATEMENTS_AGENT_ID in existing: + return None + raw = { + "name": "Monthly statements", + "kind": "plain", + # ⚠ A SCHEDULE THAT IS OFF, not an absent schedule: the cron is the CONFIGURATION T37 asks + # a person to open and read, and a blank one would make them invent it before they could + # judge it. + "schedule": {"cron": STATEMENTS_DEFAULT_CRON, "enabled": False}, + "trigger": {"key": "schedule"}, + "flow": {"actions": [{ + "id": "act_1", "kind": "send_statement", + # ⚠ TIER BLANK = every tier, which is the honest default: narrowing to "A-Urgent" would + # be us deciding who gets chased, and the blank is the value the config panel shows as + # "all of them" rather than an empty box. + "config": {"tier": "", "subject": "", "intro": "", "footer": ""}, + }]}, + } + defn, err = clean_definition(raw, None, username, rt=rt) + if err: + return None + defn["id"] = STATEMENTS_AGENT_ID + # Stamped AFTER the clean, because `clean_definition` reads this key from `prev` only — a + # payload may never assert it (see that function's note). + defn["system"] = SYSTEM_STATEMENTS + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + # ⚠ RE-CHECKED INSIDE THE MUTATION. Two concurrent list calls can both pass the read above; + # the store applies mutations under its own lock, so this is where "only once" is actually + # decided. Without it the second writer would overwrite a row a person may already have + # edited, silently resetting their schedule and template. + if STATEMENTS_AGENT_ID not in cur: + cur[STATEMENTS_AGENT_ID] = defn + return cur + + _store_update(rt, _up, flush="async") + return defn + + +def run_plain(rt, defn, username="automation", log=print, step=_no_step, rows=None): + """⭐ WAVE 24 / R6 — the runner for an automation with NO machine step: its flow IS the whole + automation. Returns the same 5-tuple every other runner does, so `run_now` needs no special + case and `apply_actions` still runs at the ONE call site that already exists. + + Before this existed, `RUNNERS.get("plain")` was None and pressing Run now committed + `error: unknown automation kind 'plain'` — on the kind every new automation now has. + + ⛔ AN UNBOUND FLOW ANSWERS `partial`, NEVER `ok`, and that is the load-bearing line here. + `apply_actions` returns immediately when there is no table, so a flow whose only action is + `create_record` into some other database does NOTHING — and a green dot over nothing is the + exact defect this module names in three other places. It reports the honest state and says + which fact is missing. + """ + # ⭐⭐ WAVE 35 · T36 — THE BATCH FLOW BRANCHES BEFORE THE TABLE CHECK, and the ORDER is the + # whole of it. A statements agent binds no `ut_*` table (its customers come from the Odoo AR + # worklist), so leaving this below the `if not table` return would answer "no database is bound + # yet" and the step would never run — a feature that is whole, gated and unreachable, which is + # this repo's most-repeated failure. See the section above `run_plain` for why it is not an + # `apply_actions` arm. + if is_statement_flow(defn): + return run_statements(rt, defn, username=username, log=log, step=step) + table = _flow_table(defn) + if not table: + return ("partial", + "no database is bound yet. A plain automation walks the records of its target " + "database, and this one names none (pick one on the trigger, or in Properties)", + {}, [], {}) + all_rows = (ut_get(rt, table) or {}).get("rows") or {} + # ⛔ THE TRIGGER'S RECORDS WIN OVER THE WHOLE TABLE, and getting this wrong is a day-one bug + # on the kind every new automation now has. `plain` has no machine step, so it has nothing of + # its own to call "the records this run touched" — and walking the WHOLE table would mean + # "When a record is CREATED in ut_leads -> set status = New" writes `New` onto every lead the + # first time one is added. The trigger knows exactly which rows fired; `run_now` threads them + # here. `rows=None` (manual, schedule, Run now) still means the whole table, which is what + # those genuinely mean. + ids = ([r for r in (rows or []) if str(r) in all_rows] if rows + else sorted(all_rows, key=_rid_num)) + + # An enrichment-only flow is the one exception to the ordinary manual-run rule above. Its + # saved View and quota are not merely how the action decides whether to spend; they ARE the + # set of records the automation was asked to process. Walking every row first meant a run + # bound to a 10-record "Pending" view announced (and needlessly visited) all 61 table rows. + # Besides being misleading, that shape made the runtime scale with unrelated historical rows. + # + # Keep this deliberately narrow: any flow with another action must still hand that action the + # whole manual/scheduled scope. Only one direct, enabled enrich action has no other record + # semantics to preserve, so it can begin at its selected records safely. + flow_actions = list((defn.get("flow") or {}).get("actions") or []) + enabled_direct = [a for a in flow_actions if isinstance(a, dict) and a.get("enabled", True)] + enrich_scope_note = "" + if rows is None and len(enabled_direct) == 1 \ + and enabled_direct[0].get("kind") == "enrich_instagram": + enrich_cfg = enabled_direct[0].get("config") or {} + profile_key = profile_field_key(ut_get(rt, table), enrich_cfg.get("profileField")) + if profile_key: + # ⛔⛔ THE SAME QUESTION MUST GET THE SAME ANSWER IN BOTH PLACES. + # `enrich_selection` is called TWICE per run — here, to decide which records the + # runner walks and announces, and again inside `_walk` to decide which the action + # SPENDS on. MEASURED LIVE 2026-08-09: this call omitted the not-found verdicts, so + # the run announced *"(1 of the 30 asked for)"* in its summary while its own note + # from the second call said *"0 of the 30"* — one run, two answers, both printed. + # ⚠ `gone` is read from the definition here rather than passed in, because this + # function has the definition and `_walk` does the same read; a third source of the + # same fact is how the two would drift again ([[one-evaluator-per-question]]). + selected, enrich_scope_note = enrich_selection( + rt, table, enrich_cfg, profile_key, + gone=dict(((defn or {}).get("state") or {}).get("enrichNotFound") or {})) + # ⛔ D-112 — AND THE CONSTANT IS THE FIX, NOT THE `if`. This guard was a `startswith` + # against the sentence SPELLED OUT, twelve hundred lines from the only place that + # produces it. Reword the message in `enrich_selection` — a perfectly ordinary edit, + # since it is a sentence a person reads — and this line silently stops matching, the + # run goes back to reporting `ok`, and the defect D-112 describes returns with nothing + # anywhere going red [[gate-pins-a-spelling-not-a-claim]], on the product side. + # ONE constant, two readers, so the wording is free to change and the behaviour is not. + if enrich_scope_note.startswith(ENRICH_VIEW_UNREADABLE): + return ("partial", f"nothing walked in {table}. {enrich_scope_note}", {}, [], {}) + ids = selected + step(f"Walking {len(ids)} record{'' if len(ids) == 1 else 's'} in {table}") + counts = {"records": len(ids)} + # ⭐⭐ WAVE 31 · T36 (D-143, owner ruling R6) — THE CAP IS THE TABLE'S, NOT THIS MODULE'S. + # + # ⛔ WHAT WAS WRONG, and it was a SECOND EVALUATOR rather than a wrong number. This line read + # `MAX_FLOW_RECORDS = MAX_UT_ROWS = 5000` — a constant mirrored from the EDITABLE substrate's + # ceiling — and applied it to every database alike. `ut_odoo_orders` is a CONNECTED source + # (32,826 rows live), and R6 is explicit that connected-source data has NO CAP: *"I thought we + # decided there is no cap in how many data from the API source … can be pulled into the app."* + # So a flow over it silently walked 5,000 of 32,826 — 15% of the records — and every total it + # produced understated the book while looking exactly like a complete run. + # + # ⭐ `core.user_tables.row_limit` IS THE ONE EVALUATOR for this question and it already answers + # the three cases (0 = read-through, None = connected and UNCAPPED, MAX_ROWS = editable). Using + # it here retires this module's private copy instead of correcting it — one question, one + # normaliser [[one-question-two-normalizers]]. `limit_report` is R6's SECOND sentence already + # expressed as data, so the sentence a person reads carries the cause and the recommended fix + # rather than just a number. + _cap = _flow_record_cap(table, rt) + if _cap is not None and len(ids) > _cap: + # The cap is DISCLOSED, never silent ([[no-unverifiable-aggregates]]): the summary names + # both numbers so "it only processed some of them" is readable rather than deducible — and + # since T36, WHY it applies and what to do about it. + total = len(ids) + ids = ids[:_cap] + return ("partial", + f"{total} records in {table}. This run walked the first {_cap}" + + _flow_cap_reason(table, rt), counts, ids, {}) + n = len(ids) + # ⭐⭐ 2026-08-07 (owner report) — SAY IT IN WORDS A PERSON CAN CHECK. + # Owner: *"how come it says that its 51 records walked in ut_beauty_influencer_leads (all)? + # wth is even ut_beauty_influencer_leads (all)??"* — and both halves of that name were ours: + # · `ut_beauty_influencer_leads` is the storage KEY. Every other surface in the product + # shows the database's LABEL ("Beauty influencer leads"); this one leaked the key. + # · `(all)` meant "every record, because a manual run has no trigger records to narrow to", + # which is unguessable from the word. It reads as a filter nobody chose. + # ⚠ THE NUMBER WAS ALWAYS HONEST and is unchanged: a `plain` automation walks its whole + # database, and a per-STEP narrowing (an enrich step's `fromView`) is a different count that + # this sentence never claimed to report. What was wrong was that nothing said so. + label = str((ut_get(rt, table) or {}).get("label") or "").strip() or table + scope = ("the records that fired the trigger" if rows + else ("the records selected for Instagram enrichment" + if len(enabled_direct) == 1 + and enabled_direct[0].get("kind") == "enrich_instagram" + else "every record. This run was started by hand, so nothing narrowed it")) + if enrich_scope_note: + scope += f" ({enrich_scope_note})" + # ⭐⭐ AND ON THE RUN, not only inside the summary sentence. `summary` is capped at 400 + # characters by `_commit_run`, and this note is the longest thing a run says — it names + # every skipped handle and what to do about them. MEASURED: a run that selects NOTHING + # (every candidate is a known-dead handle) never enters `_walk`, so the walk's own copy + # of this sentence is never produced. Without this line the explanation exists only in a + # string that is about to be truncated, on exactly the runs that look like the automation + # has stopped working. + counts[RUN_NOTES_KEY] = [enrich_scope_note] + return ("ok", f"{n} record{'' if n == 1 else 's'} walked in {label}: {scope}", + counts, ids, {}) + + +RUNNERS = {"plain": run_plain, "scrape_db": run_scrape_db, + "field_instagram": run_field_instagram, + # ⭐ WAVE 30 · T09 (D-129) — BOTH DISCOVERY KINDS MAP ONTO THE SAME FUNCTION. The kind is + # no longer chosen by which callable this dict holds; `run_discovery` reads it off the + # definition and looks the corpus up in `DISCOVERY_SPECS`. The two dicts are asserted + # key-for-key by the gate, because a kind in one and not the other is either an + # unroutable automation or a refused run. + "discover_instagram": run_discovery, + "discover_tiktok": run_discovery} + + +# --------------------------------------------------------------------------------------------- +# THE SOURCE REGISTRY (DEBT D-9's seam, wave-20 item 6b) +# --------------------------------------------------------------------------------------------- +# WHY THIS EXISTS BEFORE THERE IS A SECOND SOURCE. The vendor swap that produced it (HikerAPI → +# Bright Data) touched a dozen places: a tier name, two node ids, a config flag, a readiness +# boolean on the wire, four client strings and two gate sections. That is what a hard-coded +# vendor costs, and TikTok (D-9) would have paid it again from scratch. +# +# So a SOURCE is declared once — what it is called, which vendor answers it, and which of the +# three verbs it can do — and the surfaces read the declaration instead of naming a vendor: +# +# probe "is this configured?" -> the honest readiness bit the UI shows +# capture enrich a handle we know -> `pull_profile`-shaped +# discover find handles we do not -> the corpus query +# +# ⚠ A SOURCE DECLARES ONLY WHAT IT CAN ACTUALLY DO. `discover: None` is not a gap to fill in +# later; it is the honest statement that this source has no discovery route, and a surface that +# offers one anyway is offering a button that must refuse. +SOURCES = { + "instagram": { + # ⛔ `vendor` NAMES NO COMPANY (owner instruction 2026-08-09). It reaches operator-facing + # copy, and which provider serves a capability is a routing decision the product owns — + # `providers.py` still holds the real keys, chains and costs. + "key": "instagram", "label": "Instagram", "vendor": "Scraper", + "probe": bd_ready, + "capture": pull_profile, + "discover": bd_filter_start, + "captureKind": "field_instagram", + "discoverKind": "discover_instagram", + "tiers": TIERS, + }, + # ⭐⭐ WAVE 29 — D-9 LANDS, AND IT LANDED AS THE ONE ENTRY THIS REGISTRY PREDICTED IT WOULD. + # The canvas, the append tables, the caps and the honest-status contract were already in place; + # what TikTok added was a field map, a discovery runner and three dataset ids. + "tiktok": { + # `vendor` NAMES NO COMPANY (owner instruction 2026-08-09) — it reaches operator-facing + # copy, and which provider serves a capability is a routing decision `providers.py` owns. + "key": "tiktok", "label": "TikTok", "vendor": "Scraper", + "probe": bd_ready, + # ⭐ WAVE 30 · T08 — CAPTURE IS WIRED, and it is a LAZY reference rather than the function + # object every other row here holds. `connectors_tt` imports THIS module at module level, + # so naming `connectors_tt.pull_profile_tt` at import time is a circular import — which is + # why every existing site does `import connectors_tt` inside the function body. A + # zero-argument callable keeps the registry's shape (a reader gets a callable, not a + # string) without moving the import. + # ⚠ AND SETTING IT SWITCHES NOTHING ON. `SOURCES` is never subscripted anywhere; its one + # reader is `source_status()`, which reads `key/label/vendor/probe/discover`. An action + # kind is gated in three other places entirely (`ACTION_CATALOG`, `clean_actions`, + # `apply_actions`). This row is documentation that must not lie, not wiring. + "capture": lambda *a, **k: __import__("connectors_tt").pull_profile_tt(*a, **k), + "discover": bd_filter_start, + "captureKind": "enrich_tiktok", + "discoverKind": "discover_tiktok", + # No tier ladder: R6/R7 retired the free rung from enrichment entirely, and TikTok never + # had one. An empty tuple is the honest statement, not a gap to fill in later. + "tiers": (), + }, +} + + +def source_status(): + """`[{key,label,vendor,ready,canDiscover}]` — what the surface says about each source. + + ⚠ A BOOLEAN, NEVER THE KEY (the wire rule the `hikerReady` bit already followed): a surface + needs exactly one bit to say "the paid rung is not configured" honestly, and shipping the + credential to a browser would put a billable secret in every user's devtools. + """ + # ⛔ `vendor` IS NOT ON THE WIRE (2026-08-06). It stays in `SOURCES` as the record of which + # supplier this rung actually uses — that history is worth keeping in the code — but a + # customer has no use for it, and a server-supplied string rendered by a generic component + # is precisely the half a client-file scan cannot see. + return [{"key": s["key"], "label": s["label"], + "ready": bool(s["probe"]()), "canDiscover": bool(s.get("discover"))} + for s in SOURCES.values()] + + +# --------------------------------------------------------------------------------------------- +# THE CANVAS GRAPH (R5) — topology on the SERVER, pixels on the client +# --------------------------------------------------------------------------------------------- +# WHY THE TOPOLOGY LIVES HERE. The canvas draws what an automation DOES; the runners above are +# what it does. Two descriptions of one thing in two languages drift, and the drift is invisible — +# a canvas that still shows a step the engine stopped running looks completely fine. So the node +# list is derived from the DEFINITION by the same module that executes it, rides on the payload +# the way `CRON_PRESETS` and the kind list already do, and is asserted in `verify_automation.py`. +# The client owns layout (pixels, pan/zoom, hit targets) and nothing else. +# +# ⚠ BRANCHES RENDER, THEY DO NOT EXECUTE (R5, accepted at grill; DEBT D-7). The `capture` node is +# a real branch in BEHAVIOUR — the paid rung answers or the anonymous ladder does — but the engine +# walks it linearly. Nothing here should be read as a general branching runtime. + +#: A node's dot vocabulary. Deliberately WIDER than the run-level `STATES`: a node can be +#: `blocked` or `skipped` in ways a whole run cannot, and folding those into `partial` would throw +#: away the only information the dot is there to carry. +NODE_STATES = ("idle", "ok", "partial", "error", "blocked", "skipped") + +#: node id -> the switch it flips. A node ABSENT from this map has no switch, and that is a +#: deliberate answer rather than an omission: a "Fetch the page" step that can be turned off is +#: not an automation with a disabled step, it is a broken automation with a lie on it. The ones +#: here are all REAL — each changes what the next run does: +#: schedule the trigger fires on its cron, or only by hand +#: postMetrics likes/comments per post are bought, or the engagement series does not grow — +#: it costs a vendor record PER POST rather than per profile (measured: the +#: profile row carries post identity and no engagement) +#: commentMetrics the Comments dataset is bought, which can bill many rows per post +#: write DRY RUN — read everything, compute the counts, write nothing anywhere +#: +#: ⭐⭐ WAVE 28 / R5+R6 (contract C3) — `paid` AND `fallback` ARE GONE, and what replaced them is +#: the point of the ruling: the money switches used to be "which rung do we try" (a question about +#: our plumbing), and they are now "what do you want captured" (a question about the user's data). +#: Profile is always captured and has no switch — an automation that fetches nothing is not an +#: automation with a step turned off, it is a broken one with a lie on it. +#: ⚠ `trigger: "schedule"` IS NOT PART OF THAT COLLAPSE. It is the trigger card's own on/off and +#: has its own branch in `toggle_node` (event triggers flip themselves, not the cron); C3's "the +#: three toggles" names the CAPTURE ladder it is reshaping. +NODE_TOGGLES = {"trigger": "schedule", "capture_posts": "postMetrics", + "capture_comments": "commentMetrics", "write": "write"} + + +def _cron_label(cron): + return next((p["label"] for p in CRON_PRESETS if p["cron"] == cron), cron or "") + + +def node_status(steps, nid): + """A node's dot — from the last run's MEASURED step outcome, or `idle`. + + ⛔ NEVER INFERRED FROM THE RUN'S OVERALL STATE, and this is a module-level function precisely + so a negative control can prove that. Every run stored before W19-C has no `steps` map at all, + and painting those nodes green because the run said `ok` would manufacture a measurement — the + same defect as the green dot over the empty posts table recorded in `run_field_instagram`. + No data ⇒ `idle`. An unrecognised word ⇒ `idle`, never passed through to the UI. + """ + v = str((steps or {}).get(nid) or "") + return v if v in NODE_STATES else "idle" + + +def graph(defn): + """One automation → `{nodes, edges}`, left to right. Pure over the definition.""" + defn = defn if isinstance(defn, dict) else {} + cfg = defn.get("config") or {} + sched = defn.get("schedule") or {} + runs = defn.get("runs") or [] + last = runs[0] if runs and isinstance(runs[0], dict) else {} + steps = last.get("steps") if isinstance(last.get("steps"), dict) else {} + + def node(nid, kind, title, subtitle, col, row=0, panel="", detail="", on=True): + return {"id": nid, "kind": kind, "title": title, "subtitle": subtitle, "detail": detail, + "col": col, "row": row, "panel": panel or nid, "enabled": bool(on), + "toggle": NODE_TOGGLES.get(nid, ""), "status": node_status(steps, nid)} + + on = bool(sched.get("enabled")) + trg = defn.get("trigger") or {} + if trg.get("key") in TRIGGER_KEYS and trg.get("key") not in ("manual", "schedule"): + # C3 (wave 22): the trigger node SAYS which trigger this flow has — the board and rail + # read the same node, so an event-triggered flow must not read "Schedule". The membership + # test is the KEY SET, not a hand-listed tuple: wave 23 added four triggers and the old + # tuple would have quietly rendered every one of them as "Schedule" (they are stored, so + # the else-branch was reachable) — green, compiling, and wrong on screen. + t_on = bool(trg.get("enabled", True)) and not trg.get("paused") + sub = TRIGGER_LABELS.get(trg["key"], trg["key"]) + watched = ", ".join(trg.get("fields") or []) or "any field" + # ⭐ WAVE 24 · laws 4/5 — `event_field` no longer names a FIELD (its condition is the + # whole of it) and `record_updated` no longer names a condition (its watched fields are). + # These two lines are the migration's visible half: a node still describing a `.field` + # the validator has stopped storing would be the surface and the store disagreeing. + # ⚠ W34-T47 — A COLON, NOT A FULL STOP. R6's sweep turned these four em dashes into + # sentence breaks, which is right for a REFUSAL and wrong here: these are compact node + # SUBTITLES ("ut_leads: view v1"), not prose, and a full stop mid-label reads as two + # truncated fields rather than one qualified name. The rule the sweep encodes is "a + # sentence that used a dash usually wants a period"; a LABEL usually wants a colon. + det = {"event_field": (f"{trg.get('table', '')}" + + (f": {_lane_sentence(trg.get('when'))}" + if trg.get("when") else "")), + "record_updated": f"{trg.get('table', '')}: {watched}", + "record_created": str(trg.get("table") or ""), + "enters_view": f"{trg.get('table', '')}: view {trg.get('viewId') or 'unset'}", + "form_submitted": str(trg.get("table") or ""), + "webhook": "POST the hook URL to fire it", + # ⭐ WAVE 30 · T05 — BOTH corpus triggers, and the value is identical because the + # question is: a discovery trigger's detail line IS its filter. Without the TikTok + # key this fell to `.get(..., "")` and the trigger node rendered with a blank + # subtitle, which reads as "nothing configured" on a fully configured automation. + "ig_profile_match": (_predicate_sentence(cfg.get("predicates"), + cfg.get("operator"))[:80] + if cfg.get("predicates") + else "No filters yet. Nothing to search for"), + "tiktok_profile_match": (_predicate_sentence(cfg.get("predicates"), + cfg.get("operator"))[:80] + if cfg.get("predicates") + else "No filters yet. Nothing to search for"), + "email": str(trg.get("query") or "")}.get(trg["key"], "") + if not trg.get("configured", True): + det = "Finish setting this trigger up before it can fire" + if trg.get("paused"): + det = str(defn.get("statusNote") or "paused") + nodes = [node("trigger", "trigger", sub, + "On" if t_on else "Off", 0, panel="trigger", on=t_on, detail=det)] + else: + # ⭐ 2026-08-07 (owner ruling) — THE CARD NAMES WHAT ACTUALLY FIRES IT. With the cron off + # this node is a MANUAL trigger, and titling it "Schedule" was the screen disagreeing with + # the user's own pick — the same class of defect as the Database picker that could not be + # reached. ⚠ The SUBTITLE is untouched ("Manual only" / the cron label): it is what the + # disabled-schedule gate asserts, and it was never the wrong half. + # ⚠ `panel="schedule"` STAYS whichever way it reads. Picking Manual says how this fires + # TODAY, not that it may never be scheduled — the cron has to stay one click away, and + # this is the only node that offers it. + nodes = [node("trigger", "trigger", "Schedule" if on else "Manual", + _cron_label(sched.get("cron")) if on else "Manual only", 0, + panel="schedule", on=on, + detail="" if on else "It still runs when you press Run now")] + edges = [] + + if defn.get("kind") == "field_instagram": + # ⭐⭐ WAVE 28 / CONTRACT C3 — FOUR NODES, THREE OF THEM SWITCHES OVER WHAT IS CAPTURED. + # This branch used to draw a FORK: `Capture` splitting into `Exact counts` (paid) and + # `Estimated counts` (the free anonymous ladder), rejoining at `Post engagement`. R5 + # deleted the ladder, so the fork had one arm; keeping it would have drawn a decision the + # engine no longer makes, with a switch (`fallback`) flipping a config key the cleaners + # now discard. A canvas that offers a choice the runtime ignores is worse than no canvas. + # ⚠ `Profile set` STILL CARRIES NO SWITCH, and now that is the whole ruling rather than an + # implementation detail: the profile is always captured (R6), Posts and Comments are the + # opt-ins, and both are OFF until asked for. + metrics_on = bool(cfg.get("postMetrics")) + comments_on = bool(cfg.get("commentMetrics")) + max_posts = cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL + nodes += [ + node("source", "source", "Profile set", cfg.get("targetTable") or "No database", 1, + panel="source", + detail=f"URL column: {cfg.get('urlField')}" if cfg.get("urlField") + else "URL column: the one the field is bound to"), + # ⚠ ITS OWN NODE BECAUSE IT IS ITS OWN BILL. The profile row carries post IDENTITY + # and no engagement (measured), so likes/comments are a SECOND vendor call per post. + # A switch that multiplies a run's cost by the post count deserves to be visible on + # the canvas rather than buried in a config panel. + node("capture_posts", "capture", "Post data", + "Likes and comments per post" if metrics_on else "Off", + 2, panel="capture", on=metrics_on, + detail=(f"Up to {max_posts} posts per profile" if metrics_on else + "ut_ig_post_snapshots only grows while this is on")), + node("capture_comments", "capture", "Comment data", + "Comments on those posts" if comments_on else "Off", + 3, panel="capture", on=comments_on, + detail=("The full comments dataset. Many rows per post" if comments_on else + "Comments already embedded in a paid post row are kept either way")), + node("write", "write", "Write", + cfg.get("targetTable") or "No database", 4, panel="write", + on=not cfg.get("dryRun"), + detail="+ ut_ig_snapshots · ut_ig_posts · ut_ig_post_snapshots"), + ] + edges = [{"from": "trigger", "to": "source", "label": ""}, + {"from": "source", "to": "capture_posts", "label": ""}, + {"from": "capture_posts", "to": "capture_comments", "label": ""}, + {"from": "capture_comments", "to": "write", "label": ""}] + elif defn.get("kind") in DISCOVERY_KINDS: + # ⭐⭐ WAVE 30 · T05 — WIDENED FROM `== "discover_instagram"`, AND THIS IS THE FAILURE T04 + # UNMASKS. With the seed fixed but this arm still Instagram-only, a stored TikTok search + # fell past every arm to the no-bare-`else` promise below and returned THE TRIGGER NODE + # ALONE — so no node carried `panel="find"`, and `AutomationFind` (which mounts on + # `panelKey === "find"`) could never appear. The person would have picked the trigger + # successfully and then found nowhere to type a filter: a second, stranger bug, arriving + # as the reward for fixing the first one. + _platform, _default_table, _, _ = discovery_facts(defn.get("kind")) + limit = int(cfg.get("recordsLimit") or 0) + est = discover_estimate(limit) + pending = str((defn.get("state") or {}).get("pendingSnapshot") or "") + nodes += [ + node("find", "source", f"Find {_platform} profiles", + _predicate_sentence(cfg.get("predicates"), cfg.get("operator"))[:60], 1, + panel="find", + detail=(f"Up to {limit} profiles · about ${est['usd']}" + if bd_ready() + else "Profile search is not set up yet")), + node("collect", "capture", "Collect results", + "Searching…" if pending else "Takes about 20 minutes", 2, + panel="find", + detail=""), + node("write", "write", "Save results", + cfg.get("targetTable") or _default_table, 3, panel="write", + on=not cfg.get("dryRun"), + detail=""), + ] + edges = [{"from": "trigger", "to": "find", "label": ""}, + {"from": "find", "to": "collect", "label": ""}, + {"from": "collect", "to": "write", "label": ""}] + elif defn.get("kind") == "scrape_db": + url = str(cfg.get("url") or "") + host = (urlparse(url).hostname or url or "No URL") if url else "No URL" + which = ("Structured data (JSON-LD)" if cfg.get("extract") == "jsonld" + else f"HTML table #{int(cfg.get('tableIndex') or 0)}") + mapped = len(cfg.get("fieldMap") or {}) + nodes += [ + node("fetch", "source", "Fetch page", host, 1, panel="source", detail=url[:90]), + node("extract", "capture", "Extract", which, 2, panel="columns", + detail=f"{mapped} column{'' if mapped == 1 else 's'} mapped · key: " + f"{cfg.get('keyField') or ', '}"), + node("write", "write", "Write", cfg.get("targetLabel") or "Scraped table", 3, + panel="write", on=not cfg.get("dryRun"), + detail=cfg.get("targetTable") or "a new database"), + ] + edges = [{"from": "trigger", "to": "fetch", "label": ""}, + {"from": "fetch", "to": "extract", "label": ""}, + {"from": "extract", "to": "write", "label": ""}] + # ⛔ WAVE 24 — THERE IS NO BARE `else` HERE ANY MORE, and its absence is the point. + # It used to be `scrape_db`'s branch, which meant a kind this function had never been taught + # about inherited a scrape's three machine nodes: `Fetch page / No URL`, `Extract / HTML + # table #0`, `Write / Scraped table`, over a definition with no URL and no field map. A + # surface that draws nothing is obviously incomplete; one that draws a fetch step that does + # not exist is confidently wrong, and it is wrong on the canvas, the builder and the board at + # once because all three read this. + # + # ⭐ SO: `plain` (R6) — and any future kind, until somebody writes its arm — ANSWERS THE + # TRIGGER NODE AND NOTHING ELSE: `{"nodes": [], "edges": []}`. Exactly one node, + # guaranteed. The Builder's node filter and `stages_for` (hence the Board) both depend on + # that promise, so it is stated rather than left to be inferred from the control flow. + return {"nodes": nodes, "edges": edges} + + +def toggle_node(rt, auto_id, node_id): + """Flip ONE node's switch. Returns `(defn, error)`. + + The node → config mapping lives on the SERVER for the reason the whole graph does: a client + that knew which field each node writes would be a second copy of that knowledge, free to + drift. The canvas just says "this node was clicked". + """ + prev = all_definitions(rt).get(str(auto_id)) + if prev is None: + return None, "no such automation" + target = next((n for n in graph(prev)["nodes"] if n["id"] == str(node_id)), None) + if target is None: + return None, f"there is no {node_id!r} step in this automation" + if not target.get("toggle"): + return None, (f"the {target['title']} step cannot be turned off. It is what this " + f"automation is") + cfg, sched = dict(prev.get("config") or {}), dict(prev.get("schedule") or {}) + which = target["toggle"] + trg_key = str((prev.get("trigger") or {}).get("key") or "") + if which == "schedule" and trg_key and trg_key not in TRIGGER_SCHEDULE_KEYS: + # The trigger node's switch flips THE TRIGGER when the flow has an event one — flipping + # the cron under a node labelled "When a field changes" would be a switch that lies. + # ⭐ WAVE 24 — DERIVED from `TRIGGER_SCHEDULE_KEYS` instead of the hand-listed tuple that + # was here, which omitted `record_updated`, `enters_view` and `form_submitted` and so + # told exactly that lie for all three. The set names the SMALL side (schedule-driven), so + # a trigger added to `TRIGGER_KEYS` defaults to flipping itself, which is the safe half. + trg = dict(prev.get("trigger") or {}) + trg["enabled"] = not (bool(trg.get("enabled", True)) and not trg.get("paused")) + trg["paused"] = False # a deliberate flip clears an auto-pause: human wins + return patch(rt, auto_id, {"trigger": trg}) + if which == "schedule": + sched["enabled"] = not sched.get("enabled") + elif which == "postMetrics": + cfg["postMetrics"] = not cfg.get("postMetrics") + elif which == "commentMetrics": + cfg["commentMetrics"] = not cfg.get("commentMetrics") + elif which == "write": + cfg["dryRun"] = not cfg.get("dryRun") + return patch(rt, auto_id, {"config": cfg, "schedule": sched}) + + +# --------------------------------------------------------------------------------------------- +# THE BOARD IS DELETED (wave 27 item 12, owner ruling R3) — what is left, and why it stayed +# --------------------------------------------------------------------------------------------- +# Wave 22 made the automation detail a kanban board: columns were the flow's stages, cards were +# records of the target database, and each record's position lived in a single-select STAGE FIELD +# the engine created on the customer's own database. Wave 26 deleted the board's built-in +# terminals; wave 27 deletes the rest — `board()`, `move_card()`, `stages_for()`, +# `ensure_stage_field()`, the four card builders, the lane vocabulary on the wire, the review +# branch of the action walk, and the endings that sent a terminal card back round. +# +# ⛔ THE REASON IS A DATA REASON, NOT A UI ONE, and it is the sentence to keep: the board wrote +# MACHINE COLUMNS INTO A TENANT'S OWN TABLE — a stage select, an `_at` stamp and a `_cycles` +# counter per automation — to render a view of state the RUN LOG already holds. The customer paid +# for that in columns they did not ask for, in a store commit on every scheduled run, and in a +# permission wall (`humanMoves`/`arrive`) that existed only to stop them editing the columns we +# had added. `migrate_ig_tables` now drops those columns wherever an automation created them. +# +# ⛔⛔ `notify_review` IS GONE FROM HERE AND ITS CONSUMER IS STILL ALIVE — DEBT D-101, and this +# paragraph is its TOMBSTONE, placed where the next reader of the review machinery will look. +# +# `notify_review` was the producer of the `automation_review` notification: a card reaching a +# review stage told a human to come and look. It had ZERO callers even before R3 (wave-27 session +# C found it and correctly left it alone as another lane's subject), and R3's deletion took the +# function with the rest of the board. `verify_automation`'s W23-B section asserts its absence by +# name, so it cannot come back by accident. +# +# ⚠ THE SWEEP FOR ITS READERS, RUN 2026-08-12 (W31-T37) AND REPORTED RATHER THAN ASSUMED EMPTY: +# · SERVER — `notify_review` appears in exactly ONE file, `verify_automation.py`, inside the +# list of names asserted GONE. **No `.py` file anywhere produces an `automation_review` +# notification**; `core.alerts.notify` defaults `topic='automation'`, which is a different, +# live shape (the run-outcome alert). +# · CLIENT — the branch is FULLY ALIVE and is NOT in this fence: +# `web/src/alerts/alertsModel.ts` (`AUTOMATION_REVIEW_KIND`, `isAutomationReview`, the +# `autoId` guard) and `web/src/alerts/AlertsPane.tsx` (two call sites), plus legs in +# `web/verify_alerts.py`. All four are session B's files. +# ⇒ **The producer is deleted; the consumer guards an event that can never arrive.** D-101's own +# exit condition says these are one defect seen from two ends and must be closed in ONE change — +# so this half is the tombstone plus the sweep, and the client half is routed to B rather than +# reached across a fence. If a future wave gives review stages a real producer, THIS is the note +# that says the client already knows how to render it. +# +# ⚠ WHAT SURVIVES, AND IT IS DELIBERATE IN EACH CASE: +# * `lane_match` + `LANE_OPS` + the condition trees below — never board-only. They are the +# evaluator for ACTION conditions, trigger conditions and `find_records`, and the board was +# one of four callers. The name is the last thing the board left behind here. +# * `retire_automation_stage_fields` / `retire_automation_board_state` / `_without_retired_board` +# — the MIGRATION. It must outlive the thing it retires, or a definition written before the +# retirement walks into a runtime that no longer understands it. +# * `ai_decide` + `review_audit` — R3 keeps review "as an AI decision without lanes"; they are +# parked with no caller and say so at their own definitions. + +LANE_OPS = ("=", "!=", ">", ">=", "<", "<=", "includes", "not_includes", + "is_empty", "is_not_empty") +LANE_NULLARY_OPS = ("is_empty", "is_not_empty") + +# ───────────────────────────────────────────────────────────────────────────────────────────── +# WAVE 23 · C4 — CONDITION TREES. `Cond = leaf | {all: [Cond…]} | {any: [Cond…]}` +# +# A leaf is exactly what wave 22 called a lane condition (`{field, op, value?}`), which is why +# this generalisation needed no migration: a stored leaf IS a valid tree, `cond_match` dispatches +# on shape, and nothing at rest is rewritten until the owner saves that automation. (The doc +# calls this "legacy single-condition lanes auto-wrap as {all:[leaf]} on read" — wrapping is the +# same function applied one level up, so the cheaper honest version is to evaluate the leaf where +# it lies and never touch the bytes.) +# +# ⛔ DEPTH IS BOUNDED AND THE BOUND IS ENFORCED AT WRITE TIME, not at evaluation time. An +# unbounded tree is an unbounded evaluation on a hook that runs inside somebody's keystroke, and +# "the server got slow" is the failure nobody traces back to a filter somebody nested 40 deep. +# ⛔ REFUSE-NEVER-COERCE all the way down (`clean_predicates`' discipline): an empty group, an +# unknown comparison, a valueless compare are all REFUSED with the reason named. A group that +# quietly dropped its unanswerable leaf would WIDEN — the tri-state scar the filter engine +# carries ([[cg-filter-engine-sql-port]]), reproduced here where nothing would report it. +MAX_COND_DEPTH = 3 +MAX_COND_CHILDREN = 12 +COND_GROUP_KEYS = ("all", "any") + + +def clean_cond(raw, depth=0, where=""): + """Validate one condition TREE. Returns `(cond|None, error)`; `(None, None)` means "no + condition", which is legal everywhere a condition is optional (the catch-all lane, an + unconditioned trigger, an action group that always runs).""" + if raw in (None, "", {}): + return None, None + at = f" on {where}" if where else "" + if not isinstance(raw, dict): + return None, f"the condition{at} must be an object" + conj = [k for k in COND_GROUP_KEYS if k in raw] + if len(conj) > 1: + return None, (f"the condition{at} sets both 'all' and 'any'. A group is one or the " + f"other") + if conj: + key = conj[0] + if depth + 1 > MAX_COND_DEPTH: + return None, (f"conditions nest at most {MAX_COND_DEPTH} levels deep. " + f"the group{at} is deeper") + kids_raw = raw.get(key) + if not isinstance(kids_raw, list) or not kids_raw: + return None, f"the '{key}' group{at} needs at least one condition inside it" + if len(kids_raw) > MAX_COND_CHILDREN: + return None, (f"a group holds at most {MAX_COND_CHILDREN} conditions. " + f"the '{key}' group{at} has {len(kids_raw)}") + kids = [] + for child in kids_raw: + c, err = clean_cond(child, depth + 1, where) + if err: + return None, err + if c is None: + return None, (f"an empty condition sits inside the '{key}' group{at}. " + f"finish it or remove it") + kids.append(c) + return {key: kids}, None + field = _s(raw.get("field"), 80).strip() + op = _s(raw.get("op") or raw.get("operator"), 20).strip() + if not field: + # ⚠ WAVE 26 — the owner met this sentence repeatedly and it told them nothing they could + # act on. It states a fact about the stored tree; it never said what to DO, and the usual + # cause is not a typo but an empty field picker (the flow had no walking record to offer + # columns from). The fix belongs in the message. + return None, (f"the condition{at} names no field. Pick a column, or remove the " + f"condition") + if op not in LANE_OPS: + return None, f"{op or 'that comparison'!r} is not one of: " + ", ".join(LANE_OPS) + cond = {"field": field, "op": op} + if op not in LANE_NULLARY_OPS: + v = raw.get("value") + if v is None or (isinstance(v, str) and not v.strip()): + return None, f"give a value to compare {field!r} against{at}" + cond["value"] = v.strip() if isinstance(v, str) else v + return cond, None + + +def cond_fields(cond): + """Every field name a tree reads — the set a caller must have on the row image before the + answer means anything.""" + if not isinstance(cond, dict): + return set() + for key in COND_GROUP_KEYS: + if key in cond: + out = set() + for child in cond.get(key) or []: + out |= cond_fields(child) + return out + f = cond.get("field") + return {f} if f else set() +#: Lane labels a flow already uses for its fixed stages. A lane literally called "Review" would +#: collide with the stage the label is a choice FOR, and the board could no longer tell a +#: routed card from a gated one. +# ⚠ `tracked` / `declined` STAY RESERVED even though R6 deleted the stages that used them: the +# labels still sit in boards stored before this wave, and a user lane named "Tracked" beside a +# legacy stamped cell would read as the same lane while behaving as a different one. +def _lane_num(v): + try: + return float(str(v).replace(",", "")) + except (TypeError, ValueError): + return None + + +def lane_match(cond, row): + """Does `row` satisfy this condition TREE? None matches everything (the catch-all). + + Named for the lane that first needed it; it is now C4's whole evaluator, and every wave-22 + caller (`route_record`, `_settle_eval`, `_seed_event_state`) gained tree support by keeping + this name rather than growing a second entry point that could disagree with it. + + ⛔ REFUSE-NEVER-COERCE, per record: an ordering comparison whose either side does not parse + as a number answers False — the record simply does not enter the lane — never "treat blank + as 0", which is the `toNum(null)=0` widening the 2026-08-03 owner wave was about. A blank + cell is an unknown, and an unknown cannot be less than 10,000. + """ + if cond is None: + return True + if isinstance(cond, dict): + if "all" in cond: + return all(lane_match(c, row) for c in cond.get("all") or []) + if "any" in cond: + return any(lane_match(c, row) for c in cond.get("any") or []) + raw = (row or {}).get(cond.get("field")) + op = cond.get("op") + if op == "is_empty": + return raw in (None, "") + if op == "is_not_empty": + return raw not in (None, "") + want = cond.get("value") + if op in (">", ">=", "<", "<="): + a, b = _lane_num(raw), _lane_num(want) + if a is None or b is None: + return False + return {"<": a < b, "<=": a <= b, ">": a > b, ">=": a >= b}[op] + have_s, want_s = str(raw if raw is not None else ""), str(want if want is not None else "") + if op == "includes": + return want_s.lower() in have_s.lower() + if op == "not_includes": + return want_s.lower() not in have_s.lower() + a, b = _lane_num(raw), _lane_num(want) + same = (a is not None and b is not None and a == b) or have_s == want_s + return same if op == "=" else (not same) if op == "!=" else False + + +# ───────────────────────────────────────────────────────────────────────────────────────────── +# WAVE 23 · C4 — ACTIONS (owner ruling R3). The half of the Airtable builder that DOES things. +# +# `flow.actions` is an ordered list walked per RECORD, after the flow's machine steps have run. +# A `group` holds nested actions behind a condition — Airtable's "conditional action group", +# which the owner explicitly asked to be NESTABLE (R3 supersedes the wave-22 research brief's +# rec-8 "no nesting" verdict; that recommendation was about keeping a CANVAS legible, and this +# builder is a column, not a canvas). +# +# ⛔ THE CATALOG IS SERVER-OWNED AND INCLUDES WHAT WE HAVE NOT BUILT. `ACTION_CATALOG` carries a +# `ready` flag per kind, so the "+ Add advanced logic or action" menu paints Send email / Slack / +# Run script / Generate with AI faded with a reason instead of omitting them — the same honesty +# rule the trigger list follows, and the same enforcement: `clean_actions` REFUSES an unready +# kind with a sentence, so the faded state is a wall and not a styling choice. +MAX_ACTIONS = 20 # per automation, counting nested ones +MAX_GROUP_DEPTH = 2 # a group inside a group inside a group is a flowchart, not a flow +MAX_BRANCHES = 6 # per If / then +MAX_ACTION_VALUES = 20 # cells one update/create action may write +FIND_LIMIT_MAX = 100 +#: ⭐⭐ W31-T38 / E-4 — HOW MANY BROWSER JOBS ONE RUN MAY SUBMIT, and it belongs HERE because this +#: is the only layer that knows a record WALK is happening. `web_agent.MAX_STEPS` bounds steps per +#: JOB; nothing bounded jobs per RUN, and the web arm submits one job PER RECORD. A `web_read` on a +#: flow over the Customer grid is therefore thousands of ~10 s submissions against HF's +#: 6-concurrent cap — a run whose wall-clock is `records x 10 s` and a burst the `/hf-jobs` rules +#: exist to prevent. Session E raised it and could not fix it: the seam sees one step and cannot +#: know a walk is happening. +#: ⚠ 50 IS A STATED CHOICE, NOT A MEASUREMENT: at E's measured ~9-32 s per job that is roughly +#: 8-27 minutes of serial browsing, which is a long automation run and not a runaway one. It is +#: DISCLOSED when it binds (R6's second sentence — see the web arm), never silently applied. +#: ⛔ THE REAL FIX IS BATCHING, NOT A BIGGER NUMBER: `web_agent.run_plan(steps, ctx)` already takes +#: a list and the cost is per JOB, so a flow's web steps collected into ONE call would make this +#: ceiling nearly unreachable. That is E's own PENDING row; this bound is what stops the damage +#: until it lands. +MAX_WEB_JOBS_PER_RUN = 50 +#: ⭐⭐ W33-T58 (D-191) — HOW MANY WEB STEPS MAY SHARE ONE JOB. The comment above finally got its +#: batching, so the ceiling above now counts JOBS rather than pages and a record's consecutive web +#: steps cost one cold start between them instead of ~9 s each. +#: ⚠ 20 IS THE RUNNER'S OWN `MAX_STEPS` (`jobs/web_agent_job.py`), not a number chosen here: a plan +#: longer than that is refused by the job with `kind="too_many_steps"`, which would turn a +#: performance improvement into a whole record's worth of failed steps. Kept in step by +#: `verify_web_agent.py`, which reads both. +MAX_WEB_STEPS_PER_JOB = 20 +#: ⭐⭐ W33-T56 — HOW MANY STEPS ONE `ai_agent` ACTION MAY COMPOSE FOR ITSELF. +#: ⛔ A CEILING, NOT A TARGET, and it is the only bound between a sentence and a browser session. +#: The five `web_*` kinds are each ONE action a person wrote down; this one is a description that +#: the model turns into a journey at run time, so the number of real browser actions it performs is +#: decided by a paragraph rather than by the flow. Twelve is enough for "log in, search, open the +#: third result, read the price" and short of anything that reads as a program. +#: ⚠ Bounded by `MAX_WEB_STEPS_PER_JOB` as well, since the composed steps ride ONE job. +AI_AGENT_MAX_STEPS = 12 + + +#: The test seam for `_ai_agent_plan` — `None` in every shipped path. `verify_automation.py` sets it +#: so the fuzzy step is proven end to end with NO API key and NO spend, exactly as +#: `routes_automation._DRAFT_CHAT` does for the drafting door. +_AI_AGENT_CHAT = [None] + + +def _ai_agent_plan(cfg, row, log=print, st=None, user=""): + """A description + one record -> `(steps, "")` for `run_plan`, or `(None, sentence)`. + + ⭐⭐ W33-T56. This is the whole of the fuzzy step: the instruction and the record's own values + go to the model, and CONCRETE `web_*` steps come back — the same vocabulary a person could have + written by hand, so everything downstream (the seam, the job, the checkpoint, the per-step + verdicts) is unchanged and none of it has to know an assistant was involved. + + ⛔ IT MAY COMPOSE ONLY BROWSER STEPS. `ai_review.draft_flow` is handed a catalog filtered to + `WEB_KINDS`, so the enum in the tool schema cannot express `create_record` or a connector call. + A fuzzy sentence therefore cannot be talked into writing the tenant's data: the worst a bad + instruction can do is waste one browser session. That is a property of the SCHEMA, not of the + prompt, which is the only version of it worth relying on. + ⛔ AND IT IS BOUNDED. `maxSteps` is clamped at save time and re-applied here, because the + number of real browser actions is otherwise decided by a paragraph. + ⚠ `interpolate` FIRST, so the model is told about THIS record — `{{Website}}` is a different + page for every row, and a journey composed against the template would be one guess repeated. + """ + import ai_review # noqa: PLC0415 + url = interpolate(str(cfg.get("url") or ""), row) + instruction = interpolate(str(cfg.get("instruction") or ""), row) + cap = max(1, min(int(cfg.get("maxSteps") or AI_AGENT_MAX_STEPS), MAX_WEB_STEPS_PER_JOB)) + web_only = [r for r in ACTION_CATALOG if r.get("kind") in WEB_KINDS and r.get("ready")] + draft, why, _prov = ai_review.draft_flow( + prompt=(f"Starting page: {url}\n\nDo this: {instruction}\n\n" + f"Answer with at most {cap} steps. The FIRST step must open the starting page."), + catalog=web_only, + required={k: v for k, v in ACTION_REQUIRED.items() if k in WEB_KINDS}, + triggers=[], tables=[], + # ⭐ W35 · C7 (`NOTE E-16`) — threaded IN from `_walk` rather than resolved here: this + # function is pure over `(cfg, row)` by design and giving it a store handle of its own + # would be a second way to reach the tenant. + st=st, user=user, + chat=_AI_AGENT_CHAT[0]) + if why or not draft: + return None, (why or "the assistant produced no steps for that instruction") + steps, seen_url = [], False + for i, a in enumerate(draft.get("actions") or [], 1): + kind = str(a.get("kind") or "") + if kind not in WEB_KINDS: + # Belt and braces behind the enum: a rung that ignores its own schema is stopped here + # rather than reaching the browser. + return None, f"the assistant asked for a step this action cannot perform ({kind})" + c = a.get("config") or {} + step = {"kind": kind, "id": f"ai{i}", + "url": str(c.get("url") or "") or (url if not seen_url else ""), + "selector": str(c.get("selector") or ""), + "attr": str(c.get("attr") or "text"), + "all": bool(c.get("all")), + "waitFor": str(c.get("waitFor") or "") or None, + "timeoutMs": int(cfg.get("timeoutMs") or WEB_READ_TIMEOUT_MS)} + if c.get("value"): + step["value"] = str(c.get("value")) + if c.get("hint"): + step["hint"] = str(c.get("hint")) + if cfg.get("dryRun"): + step["dryRun"] = True + seen_url = seen_url or bool(step["url"]) + steps.append(step) + if len(steps) >= cap: + break + if not steps: + return None, "the assistant produced no steps for that instruction" + # ⛔ THE SEAM REFUSES A JOURNEY WHOSE FIRST STEP CARRIES NO ADDRESS — there is no page to act + # on yet — and the action's own `url` is exactly the answer. Supplying it here beats letting + # the whole job be refused for a sentence the model happened not to repeat. + if not steps[0].get("url"): + steps[0]["url"] = url + return steps, "" + + +def _web_missing(kind, cfg): + """Which of this kind's REQUIRED config keys are blank — the human phrases, in table order. + + ⚠ PER KIND, because they do not need the same things: `web_goto` needs a url and no selector; + `web_fill` needs a value nobody else takes; only `web_read` needs a column to write into. One + shared three-field test would have blocked every `web_goto` ever configured for want of a + selector it does not use. + ⭐ Lifted out of `apply_actions` at W33-T58 so the BATCH BUILDER and the per-step arm ask the + same question of the same table. Two copies of "is this step configured" would let a batch + include a step the arm then refuses — or worse, exclude one it would have run. + """ + return [n for n, key in WEB_REQUIRED.get(kind, ()) if not str((cfg or {}).get(key) or "").strip()] +#: ⭐⭐ W31 QA — THE FIVE WEB KINDS THE ENGINE DISPATCHES, and this is the THIRD list that names +#: them (`web_agent.RUNNABLE_KINDS` is the seam's, `jobs/web_agent_job.py::RUNNABLE_KINDS` is the +#: runner's). ⛔ IT IS DELIBERATELY NOT AN IMPORT: `_web_agent()` resolves the seam LAZILY so an +#: absent module cannot break the engine, and a module-level `from web_agent import RUNNABLE_KINDS` +#: would throw that property away for a tuple of five strings. The three lists are held in step by +#: `verify_web_agent.py::section_one_kind_only`, which reds if any pair disagrees — a per-kind +#: allow-list in three places that CAN disagree is the defect, whichever way it points. +WEB_KINDS = ("web_read", "web_goto", "web_fill", "web_click", "web_repair") +#: What each kind cannot run WITHOUT. Absence is never refused at SAVE (a step is addable before it +#: is configured — see `_clean_action_config`'s web arm); it is refused at RUN with a sentence +#: naming what is missing. `url` is required only by `web_goto`: every other kind acts on the page +#: the flow is already on, and the runner supplies its own refusal when it genuinely needs one. +WEB_REQUIRED = { + "web_read": (("a URL", "url"), ("a CSS selector", "selector"), + ("a column to write into", "field")), + "web_goto": (("a URL", "url"),), + "web_click": (("a CSS selector", "selector"),), + "web_repair": (("a CSS selector", "selector"),), + "web_fill": (("a CSS selector", "selector"), ("a value to type", "value")), + # ⭐⭐ W33-T56 — the fuzzy step needs a page to start on and a job to do, and nothing else. + # ⚠ NO `selector` AND NO `field`: working out the selector IS the step, and where to put the + # answer is optional (a journey may only need to have been performed). + "ai_agent": (("a URL", "url"), ("a description of what to do", "instruction")), +} +#: ⭐⭐ WAVE 32 · T45 (owner item 10) — WHAT EACH ACTION KIND CANNOT RUN WITHOUT, for every kind. +#: +#: `WEB_REQUIRED` above already WAS this table for five kinds, complete with the human phrase each +#: refusal says out loud, so this is its widening rather than a second opinion — spread in, never +#: re-typed, or a wave that adds a web key would teach the run refusal and not the label. +#: +#: ⛔ MOST KINDS ARE ABSENT, AND THE ABSENCES ARE THE INTERESTING PART. `create_record`, +#: `update_record`, `find_records` and `group` are REFUSED AT SAVE when their config is incomplete +#: (`_clean_action_config`: *"the create record action names no database"*, *"…writes no values"*, +#: *"a branch with no actions inside it does nothing"*), so a stored one is configured by +#: construction and a row here would be a second, weaker copy of a wall that already holds. +#: +#: ⛔⛔ AND `enrich_instagram` / `enrich_tiktok` ARE DELIBERATELY ABSENT DESPITE BEING THE OBVIOUS +#: CANDIDATE. Their `profileField` may be empty on purpose: the C3 profile FLAG on the target +#: database resolves the binding at run time (`profile_field_key`), so an empty one is a working +#: action on any flagged database and marking it Unconfigured would put a red label on the +#: commonest correct configuration there is. The genuinely unbound case still fails closed at RUN +#: with its own sentence — which is the honest division of labour: this table holds what can be +#: answered from the ACTION ALONE, and anything needing the target database's schema stays a +#: run-time refusal rather than becoming a store read on the automations LIST path (W30-T12 took +#: that read off this route; putting it back to draw a label would undo the wave before this one). +ACTION_REQUIRED = dict(WEB_REQUIRED) +#: ⭐ W31-T38 (C5) — the web step's own clamps. `20000` matches the default E's seam documents; +#: the ceiling exists because this blocks the record walk (~9-32 s of cold start ALREADY), and an +#: automation that can be configured to wait ten minutes per record is a stalled run, not a slow one. +WEB_READ_TIMEOUT_MS = 20000 +WEB_READ_TIMEOUT_MAX_MS = 120000 +#: ⭐ WAVE 24 · C-ACT — THE MENU'S GROUP ORDER IS THE SERVER'S. The client sorts by `groupOrder` +#: and never by a literal list of group names: a client-side ordering is a second copy of this +#: table, and the way it fails is that a group added here renders last, or not at all, with +#: nothing red anywhere. +ACTION_GROUP_ORDER = {"Web action": 1, "Database": 2, "Connected": 3, "Advanced logic": 4} +ACTION_CATALOG = [ + # ── 1. WEB ACTION (owner rulings R1-R5) — DECLARED HERE, NOT YET BUILT (DEBT D-51). ──────── + # ⛔ NOT A TEASE. `clean_actions` refuses an unready kind with a sentence, so the faded row + # is a WALL, not a styling choice — and each `detail` says the true reason rather than a + # placeholder, because "coming soon" on five rows is how a menu stops being believed. + # + # ⚠ WAVE 25 RETARGETED THE ONE STRING THAT NAMED A WAVE, and the reason generalises: W24 + # scheduled this build for wave 25 and wave 25's six owner items do not include it, so + # "building next (wave 25)" became false the moment this wave shipped — a menu that dates its + # own promises has to be re-read by whoever misses the date, and a stale date is worse than + # none because it reads as a commitment somebody already broke. The replacement names the + # missing CAPABILITY, like its four siblings always did; D-51 carries the schedule. + # ⭐⭐ WAVE 31 · T38 + QA (C5) — ALL FIVE ROWS ARE READY, AND THE RULING IS WHY. + # + # ⛔ THIS REVERSES T38's ORIGINAL `how:`, ON THE OWNER'S OWN WORDS. T38 shipped `web_read` + # alone and held the other four at `ready:False`, citing PRD R10 ("the ones that WRITE to a + # third party do not flip without R5's approval gate existing"). **R5 was REVOKED the same + # day, and the revocation is recorded as an amendment on this wave's board** + # (`TICKETS.md:1418`, `mailbox/E.md:278`, `web_agent.py:103`) — owner, verbatim: + # *"make sure we unblock all web actions, we don't need approval step first wtf, I never ask + # for that."* Session C declined E's ask against the superseded half of R10, so four actions + # the owner asked for shipped unusable, and `verify_web_agent` was RED on correct-by-ruling + # code for the whole wave. Confirmed at QA against four independent recordings of the quote. + # ⚠ THE FADED ROW IS A WALL, NOT A STYLE: `clean_actions` refuses an unready kind with a + # sentence, so this flag is what makes the action storable at all — which is exactly why a + # row left `False` against a ruling is a feature that does not exist. + # ⚠ THE `detail` STRINGS CHANGED WITH THE FLAGS, deliberately. They said *"Needs the browser + # job"* and *"Needs the recorder that captures a selector"* — both false since E's runner + # landed. A menu that dates its own promises is this module's own recorded complaint one + # comment up; a row that advertises a missing prerequisite it already has is the same defect. + # ⭐⭐ WAVE 33 · W33-T56 (owner item 7, ruling R3) — THE FUZZY STEP. + # ⛔ IT ADDS; IT REPLACES NOTHING. R3 is explicit: *"No existing `web_*` kind is removed — + # dropping a kind from the catalog 400s every stored automation using it, forever"* (D-65). + # This is the step for a journey somebody can DESCRIBE but not spell as a selector; the five + # kinds below stay exactly as they are for the journeys they can already express, and a person + # who knows the selector should still use `web_read`, which costs no model call. + # ⭐⭐ WAVE 34 · W34-T44 / R18 — SIX ROWS BECAME ONE, AND THE OTHER FIVE DID NOT LEAVE THE + # CATALOG. Owner, verbatim: *"Remove Do this on a page / Open a page / Fill a field / Click + # something / Read from the page / Repair a broken step completely. One action, Web agent."* + # + # ⛔⛔ WHY THEY ARE HIDDEN AND NOT DELETED, AND IT IS NOT D-65 THIS TIME — IT IS WORSE. + # `_ai_agent_plan` builds the model's tool schema as + # `[r for r in ACTION_CATALOG if r["kind"] in WEB_KINDS and r["ready"]]` + # so DELETING these five rows empties that list, and the ONE action the ruling keeps would be + # left able to compose nothing at all. Deleting the six would have deleted the survivor, in + # silence, and `ai_agent`'s own tests would still pass because they stub the chat rung + # (`_AI_AGENT_CHAT`). D-65's usual argument (a deleted kind 400s every stored automation + # forever) applies as well and is the smaller half. + # + # ⭐ SO THE LINE MOVED FROM "IS IT IN THE CATALOG" TO "IS IT ON THE MENU". `menu: False` is + # withheld by `action_catalog()` from the picker, while `clean_actions` (which reads this + # constant, not the wire) still validates the kind, the runner still runs it, and a stored + # automation built before today keeps working and stays editable. + # ⚠ THE LABELS CHANGED TOO, and that is R18's "appear NOWHERE" clause taken literally: a + # hidden row's label is still rendered on a STORED step's card, so leaving the six captions in + # place would have kept them on screen for exactly the people who already use them. They now + # name the mechanism instead, and no old caption survives as a substring in any case. + {"kind": "ai_agent", "label": "Web agent", "group": "Web action", "ready": True, + "detail": "Describe a job on a website in words; the agent works out the steps, runs them in " + "a browser and reports what it actually did"}, + {"kind": "web_goto", "label": "Web agent step (navigate)", "group": "Web action", + "ready": True, "menu": False, + "detail": "Opens a page in a browser job and reports the title it landed on"}, + {"kind": "web_fill", "label": "Web agent step (type)", "group": "Web action", + "ready": True, "menu": False, + "detail": "Types a value into a field. Mark it secret and the value is masked in the log"}, + {"kind": "web_click", "label": "Web agent step (click)", "group": "Web action", + "ready": True, "menu": False, + "detail": "Clicks the element a selector names, and reports where it landed"}, + # ⚠ `"kind": "web_read"` AND `"ready": True` STAY ON ONE LINE. `verify_wiring`'s C5 row matches + # `"kind": "web_read".*?"ready": True` without DOTALL, so wrapping this row the way its four + # siblings are wrapped turns that cross-fence assertion red — on a formatting change, with the + # mount and the flag both intact. The row is A's file and its CLAIM is right (a catalog kind + # the client can add must be one the runner will execute); the layout is what it happens to + # depend on, so the layout is preserved here rather than the assertion weakened there. + {"kind": "web_read", "ready": True, "label": "Web agent step (extract)", + "group": "Web action", "menu": False, + "detail": "Reads one value off a live page in a browser job. Expect ~10-30 s per step"}, + {"kind": "web_repair", "label": "Web agent step (relocate)", "group": "Web action", + "ready": True, "menu": False, + "detail": "Follows a label to its control when a selector has gone stale, and proposes one"}, + # ── 2. DATABASE ────────────────────────────────────────────────────────────────────────── + {"kind": "update_record", "label": "Update record", "group": "Database", "ready": True, + "detail": "Write values onto the record walking the flow"}, + {"kind": "create_record", "label": "Create record", "group": "Database", "ready": True, + "detail": "Add a row to another database"}, + {"kind": "find_records", "label": "Find records", "group": "Database", "ready": True, + "detail": "Look rows up by condition; the run log opens them"}, + # ── 3. CONNECTED ───────────────────────────────────────────────────────────────────────── + # ⭐ WAVE 25 · C4 (owner rulings R3/R4) — THE ENRICH ACTION, and it REPLACES a whole KIND. + # `field_instagram` was an automation you created to fill one column; this is a step any flow + # can take, which is the shape it should always have had — enriching a profile is something + # you do TO a record, not a species of automation. + # ⭐ WAVE 27 · C4 (item 33) — `connector` NESTS THIS ROW UNDER "Scraper" in the action menu, + # exactly as `TRIGGER_CONNECTOR` already nests the trigger picker. See `ACTION_CONNECTOR`. + {"kind": "enrich_instagram", "label": "Enrich Instagram profile", "group": "Connected", + "ready": True, "connector": "scraper", + "detail": "Fill this record's Instagram columns from its profile, and add a point to its " + "history"}, + # ⭐⭐ WAVE 30 · T08 / CONTRACT C3 — THE SECOND NETWORK, AND THIS ROW IS THE LAST SWITCH THAT + # LANDS, deliberately. `apply_actions._walk`'s kind dispatch has NO terminal `else` (measured + # from the AST, not read): an unknown kind is walked, counted, reports the run `ok`, and + # writes nothing. So a catalog entry ahead of its runner arm would ship an action that is + # addable, clickable, storable and silently inert — strictly worse than the state before it, + # where `clean_actions` refuses the kind with a sentence. + # ⚠ `connector: "scraper"` NESTS IT UNDER THE SAME BUCKET AS INSTAGRAM (R3), which is the same + # value T07 gave the trigger — one word, both menus. + {"kind": "enrich_tiktok", "label": "Enrich TikTok profile", "group": "Connected", + "ready": True, "connector": "scraper", + "detail": "Fill this record's TikTok columns from its profile, and add a point to its " + "history"}, + {"kind": "send_email", "label": "Send email", "group": "Connected", "ready": False, + "detail": "Needs a send scope on the Gmail connection"}, + # ⭐⭐ WAVE 35 · T35 / CONTRACT C8 / OWNER RULING R10 — STATEMENTS BECOME AN AGENT STEP. + # + # Owner item 14: move Statements out of Settings and into the agent automation, "templatic and + # easily toggleable". R10 is the half that decides the shape: the step ASSEMBLES and PARKS a + # batch in a review stage, and **nothing sends without a human click**. So this row's promise + # is deliberately "prepares", not "sends" — the label a person reads must not describe an act + # the step does not perform. + # + # ⛔ ADDED BESIDE `send_email`, NEVER BY REPURPOSING IT (the ticket's own trap, D-295): a + # stored automation naming a kind that no longer exists is refused FOREVER rather than dropped + # with a reason, so adding a kind is cheap and re-pointing one is not. + # + # ⛔ `ready: True` IS WHAT MAKES IT STORABLE — `clean_actions` refuses any row whose `ready` is + # false — and the note at `enrich_tiktok` above is the reason the RUNNER ARM lands in the same + # wave rather than after it: `_walk` has no terminal `else`, so a catalog row ahead of its arm + # is addable, storable and SILENTLY INERT, which is worse than not existing. T35 lands an arm + # that reports it is not configured; T36 makes it park a real batch. + # + # ⚠ TENANT-GATED, not `ready: False`: `TENANT_GATED_ACTIONS` withholds this row from every + # tenant but #0, because the send client behind it is env-credentialed and is tenant #0's. + {"kind": "send_statement", "label": "Prepare customer statements", "group": "Connected", + "ready": True, + "detail": "Assemble this month's statements and park them for review. Nothing is sent until " + "somebody opens the batch and clicks Send"}, + # ⭐⭐ WAVE 36 · W36-T39 — D-277 CLOSED THE WAY THAT ROW ITSELF RECOMMENDED: *"an + # `ACTION_CATALOG` row with `menu: false` — one line, reusing the door R18 built this same wave + # to keep the five web kinds readable but unofferable. (a) looks right; it is E's file."* + # + # ⛔ THE DEFECT WAS A KIND IN NO CATALOG ON EITHER SIDE. `routes_automation._field_agent_rows` + # emits `flow.actions[0].kind == "ai_enrich"` for every AI-enrichment column in the tenant, and + # that string appeared ZERO times here and ZERO times in `aios-web/web/src/automation/`. The + # builder resolves a stored step's caption with `catalog.find(c => c.kind === a.kind)` and falls + # back to `a.kind`, so a field agent opened in the Agents module showed a RAW TOKEN. + # + # ⚠ `menu: False`, NEVER `ready: False`. `ready: False` renders as "coming soon" — a promise — + # and `clean_actions` refuses the kind outright, which would 400 the synthetic row the moment + # anything validated it. `menu: False` is the exact shape R18 built: withheld from the picker, + # still resolvable as a caption, still valid. + # ⛔ AND IT IS NOT ADDABLE BY HAND ON PURPOSE. An enrichment belongs to a COLUMN; the automation + # canvas is not where one is created, which is why `patch_automation` already refuses a + # `field:` id with "change its prompt, model or schedule on the column itself". + # ⭐⭐ AND A THIRD INSTANCE, FOUND BY W36-T39's OWN GATE ON ITS FIRST RUN. `_odoo_sync_row` + # emits `flow.actions[0].kind == "odoo_sync"` for the connector's synthetic schedule agent, and + # that kind was in no catalog either — the same raw token on the same screen as D-277, one row + # down. Two known instances were enough to justify the check; the check then produced a third + # nobody had booked, which is the difference between a gate and a regression test. + {"kind": "odoo_sync", "label": "Sync from Odoo", "group": "Connected", + "ready": True, "menu": False, + "detail": "Pull the connected Odoo databases on a schedule. Configured on the connector, not " + "here"}, + {"kind": "ai_enrich", "label": "Enrich this column with AI", "group": "Connected", + "ready": True, "menu": False, + "detail": "Fill an AI column for the records this flow walks. Configured on the column, not " + "here"}, + {"kind": "slack", "label": "Send Slack message", "group": "Connected", "ready": False, + "detail": "Needs the Slack connector"}, + # ── 4. ADVANCED LOGIC ──────────────────────────────────────────────────────────────────── + # `group` is relabelled "If / then" (C-ACT): "Conditional logic" described the mechanism, + # and R8 made it a FORK with lettered branches, which is a thing people already have a name + # for. The KIND is untouched — renaming it would orphan every stored action for a caption. + {"kind": "group", "label": "If / then", "group": "Advanced logic", "ready": True, + "detail": "Send the record down one of several branches, by condition"}, + {"kind": "repeating_group", "label": "Repeating group", "group": "Advanced logic", + "ready": False, "detail": "Run the same actions on every item in a list"}, + {"kind": "run_script", "label": "Run script", "group": "Advanced logic", "ready": False, + "detail": "Not built. A sandbox is its own decision"}, + {"kind": "generate_ai", "label": "Generate with AI", "group": "Advanced logic", + "ready": False, "detail": "Needs an AI action implementation"}, +] + + +#: ⭐ WAVE 27 · C4 (owner item 33) — WHICH CONNECTOR AN ACTION BELONGS TO, so the action menu can +#: nest exactly as the trigger picker already does (`reference/Airtable Automation 10.png`: an +#: "Integrations" header, one row per connector, a chevron into that connector's own submenu). +#: +#: ⚠ SAME SHAPE AS `TRIGGER_CONNECTOR`, DELIBERATELY, and the same warning applies: these are +#: GROUPING HANDLES for the picker, not `/connectors/directory` slugs. A client must group by the +#: key and render the label, never join it against the directory. +#: ⛔ AND THE LABEL IS NOT DECLARED HERE AT ALL — it is LOOKED UP in `TRIGGER_CONNECTOR` by key. +#: A "Scraper" nest in the trigger menu and a "Scrapers" nest in the action menu would be one +#: connector wearing two names on two screens somebody sees within a second of each other, and a +#: second literal is how that happens. This tuple says only WHICH connectors an action may name; +#: what they are CALLED has exactly one source. +ACTION_CONNECTORS = ("scraper",) + + +def _connector_meta(key): + """`{key, label}` for a connector key, from the one place either picker declares it.""" + key = str(key or "") + if key not in ACTION_CONNECTORS: + return None + return next((dict(v) for v in TRIGGER_CONNECTOR.values() if v.get("key") == key), None) + + +#: ⭐⭐ WAVE 35 · T35 / CONTRACT C8 — THE TENANT PREDICATE `routes_statements._royal_only` USES, +#: LIFTED SO THERE IS EXACTLY ONE COPY OF IT. C8's words are "the predicate is imported, never +#: re-expressed", and this is the direction that import can run: `automation_engine` imports no +#: route module and no FastAPI (checked), and breaking that to reach `_royal_only` would drag +#: `deps` + `routes_admin` into the engine. So the ENGINE holds the test and the ROUTE calls it. +#: +#: ⛔⛔ THIS IS NOT `odoo_relational.is_royal`, AND THE DIFFERENCE IS A SEND. That one asks "is this +#: tenant ENTITLED to Odoo databases" over `RI_SLUGS = ("", "royal-imports")` — it answers TRUE for +#: the EMPTY slug and lower-cases its input. This one is `_royal_only`'s exact test: the runtime's +#: own `key`, matched exactly. A tenant whose key never got set would pass `is_royal("")` and then +#: queue statements THROUGH TENANT #0'S ENV-CREDENTIALED SEND CLIENT, i.e. email another company's +#: customers over Royal Imports' name. Entitlement to READ is not authority to SEND, and the two +#: questions keep their two predicates on purpose. Do not "unify" them. +ROYAL_TENANT_KEY = "royal-imports" + + +def is_statement_tenant(rt): + """Exactly `routes_statements._royal_only`'s test, as a boolean over the RUNTIME. + + Keyed on the runtime, never on a request field: a tenant is a property of the SESSION, so a + payload cannot argue its way into another company's sender. + """ + return getattr(rt, "key", None) == ROYAL_TENANT_KEY + + +#: Kinds only SOME tenants may see or store, as `{kind: predicate(rt) -> bool}`. +#: +#: ⛔ FAIL-CLOSED ON `rt=None`, AND THAT IS THE WHOLE SAFETY ARGUMENT. Every reader below treats an +#: absent runtime as "not allowed", so a call site that forgets to pass one makes the action VANISH +#: rather than become universal. The opposite default would mean any future caller of +#: `action_catalog()` or `clean_actions()` silently offers tenant #0's send door to every tenant — +#: an omission that is invisible in review and loud only in production [[default-must-pass-its-own-guard]]. +TENANT_GATED_ACTIONS = {"send_statement": is_statement_tenant} + +#: The dunning buckets a statements step may filter on. ⚠ ONE LITERAL, TWO READERS: this and +#: `routes_statements.statements()`'s `"tiers"` field are the same four strings, and a step +#: configured against a tier the sender's worklist does not produce would filter to nothing and +#: report success. `routes_statements` imports this rather than repeating it. +STATEMENT_TIERS = ("A-Urgent", "B-Active", "C-Light", "Monitor") + + +def _tenant_may_use(kind, rt): + """May this runtime see/store this action kind? Ungated kinds are always yes.""" + gate = TENANT_GATED_ACTIONS.get(str(kind or "")) + return True if gate is None else bool(rt is not None and gate(rt)) + + +def catalog_kinds(): + """Every action kind this module knows about — the LABEL vocabulary. + + ⭐ D-277's WHOLE LESSON IN ONE SENTENCE: the client resolves a stored step's caption out of the + catalog and falls back to the raw kind token, so a kind the server can EMIT and the catalog + does not carry is a token on somebody's screen. This is the set that must cover every kind any + server path can put into a `flow.actions` entry, whether or not a person may add it. + """ + return frozenset(str(row.get("kind") or "") for row in ACTION_CATALOG) + + +def configurable_kinds(): + """The kinds a PERSON may add from the picker, and must therefore be able to configure. + + ⭐⭐ W36-T39 — THE CONTRACT BETWEEN THE TWO TREES, DERIVED AND NEVER LISTED. `ready` alone is + the wrong set: the five `web_*` kinds are ready and `menu: False` (R18), and `ai_enrich` is + ready and `menu: False` (D-277) — all six are real, runnable, captioned, and unofferable. What + a client must be able to CONFIGURE is exactly what a person can ADD. + + ⛔ DERIVED FROM THE CATALOG, so a new row joins the contract by existing. A hand-kept list + would be a third copy of the vocabulary, and the two copies this ticket exists to reconcile + were already one too many. + """ + return frozenset(str(row.get("kind") or "") for row in ACTION_CATALOG + if row.get("ready") and row.get("menu", True) is not False) + + +def action_catalog(rt=None): + """The catalog as the wire carries it — a copy, because a caller that mutated the module + constant would change every later reader's answer. + + ⭐⭐ WAVE 35 · T35 (C8): `rt` filters TENANT-GATED rows out entirely — not `ready: False`, not + `menu: False`, but ABSENT. A tenant that may not send statements should not learn that the + capability exists, and `ready: False` renders as "coming soon", which is a promise we are not + making to them. ⚠ Called with no `rt` the gated rows are withheld (fail-closed): see + `TENANT_GATED_ACTIONS`. + + ⭐ WAVE 24 (C-ACT): each row is stamped with its `groupOrder`, DERIVED from + `ACTION_GROUP_ORDER` rather than hand-written per row, so a group cannot be given two + different orders by two rows that claim to be in it. A group nobody has ordered sorts LAST + (not first) — a new group appearing above "Web action" because its order defaulted to 0 is + the failure that would look deliberate. + + ⭐ WAVE 27 (C4): a row naming a `connector` is stamped with the full `{key, label}` the client + nests on — RESOLVED here rather than written out per row, for the reason `groupOrder` is: two + rows in one nest cannot disagree about what that nest is called. A row naming an unknown + connector loses the stamp instead of inventing a nest with a raw slug for a title. + + ⭐⭐ WAVE 34 · W34-T44 / R18: `menu` is stamped on EVERY row, never left absent. A row the + picker must not offer carries `menu: False` and STILL RIDES THE WIRE, because the client + resolves a STORED step's label out of this same list (`AutomationBuilder`: + `catalog.find(c => c.kind === a.kind)` then `row?.label || a.kind`) — withholding the row + entirely would make an existing web step render its raw kind token at somebody, which this + module forbids in those words elsewhere. + ⚠ STAMPED RATHER THAN LEFT TO DEFAULT: absent-means-true is a rule two codebases have to + remember the same way, and a client filter is one `!== false` away from meaning the opposite. + An explicit boolean on every row cannot be read two ways. + """ + last = max(ACTION_GROUP_ORDER.values()) + 1 + out = [] + for a in ACTION_CATALOG: + if not _tenant_may_use(a.get("kind"), rt): + continue + row = {**a, "groupOrder": ACTION_GROUP_ORDER.get(a.get("group"), last), + "menu": bool(a.get("menu", True))} + conn = _connector_meta(a.get("connector")) + if conn: + row["connector"] = dict(conn) + else: + row.pop("connector", None) + out.append(row) + return out + + +def action_needs(action): + """⭐⭐ WAVE 32 · T45 (owner item 10) — what THIS action is still missing, in the words a person + reads. `[]` means Configured. + + Pure over `(kind, config)` — no runtime, no store read. That is what lets `_wire` stamp every + action on the automations LIST without putting the `user_tables` document back on a route + W30-T12 just took it off. + """ + action = action or {} + cfg = action.get("config") if isinstance(action.get("config"), dict) else {} + return [name for name, key in ACTION_REQUIRED.get(str(action.get("kind") or ""), ()) + if not str(cfg.get(key) or "").strip()] + + +def _walk_actions(actions): + """Every action in a flow, INCLUDING the ones nested inside If / then branches. + + ⛔ A FLAT `for a in actions` MISSES HALF A FLOW. `group.config.branches[].actions` is where a + conditional puts its real work, and a configured-check that only saw the top level would + report a flow ready to run while the step inside branch B had never been filled in — which is + the exact class of defect this ticket exists to surface, hiding inside the ticket's own fix. + """ + for action in actions or []: + if not isinstance(action, dict): + continue + yield action + for branch in ((action.get("config") or {}).get("branches") or []): + if isinstance(branch, dict): + yield from _walk_actions(branch.get("actions")) + + +def unconfigured_actions(defn): + """Every ENABLED action of this automation that cannot run, as `[{id, kind, label, needs}]`. + + ⚠ DISABLED ACTIONS ARE SKIPPED, and that is the point of being able to disable one: a step + somebody switched off is not a step blocking the run. `apply_actions` already ignores them. + """ + out = [] + for action in _walk_actions(((defn or {}).get("flow") or {}).get("actions")): + if not action.get("enabled", True): + continue + needs = action_needs(action) + if needs: + out.append({"id": str(action.get("id") or ""), "kind": str(action.get("kind") or ""), + "label": _action_label(action), "needs": needs}) + return out + + +def run_refusal(defn): + """⛔ WAVE 32 · T45 — the sentence a run is refused with, or `""`. + + Owner item 10: *"running an automation with ANY unconfigured action is REFUSED with a message + naming which action"*. Naming it is half the requirement and the half that is easy to drop — a + bare *"an action is not configured"* on a twelve-step flow is a hunt, not a message. + + ⛔ IT LIVES ON THE ENGINE, NOT ON THE ROUTE, BECAUSE THE ROUTE IS NOT THE ONLY DOOR. The tick + runs automations on a schedule, the webhook door runs them, and a check mounted in + `POST /automations/{id}/run` alone would refuse the button and let the cron sail past it — + [[seal-the-transport-not-the-rung]], and D-112's own shape (an action bound to a deleted view + walked zero records and reported `ok`). `run_now` refuses for every caller; the route asks + first only so the person clicking gets a 400 with the sentence instead of a silent no-op. + + ⚠ THIS IS A BEHAVIOUR CHANGE FOR STORED AUTOMATIONS AND IT IS THE RULING. A flow with one + unconfigured web step used to run, skip that step with a note, and report `ok`; it now does not + run at all. That is what "blocks the run" means, and the alternative — running everything else + and reporting success — is precisely what the owner is asking to stop. + """ + missing = unconfigured_actions(defn) + if not missing: + return "" + return "; ".join(f"{m['label']} still needs " + ", ".join(m["needs"]) for m in missing) + + +def _action_label(act): + for row in ACTION_CATALOG: + if row["kind"] == act.get("kind"): + return row["label"] + return str(act.get("kind") or "Action") + + +def clean_actions(raw, depth=0, _seen=None, _count=None, notes=None, rt=None): + """Validate `flow.actions`. Returns `(actions, error)` — refuses, never coerces. + + ⭐⭐ WAVE 35 · T35 (C8) — `rt` IS THE TENANT WALL ON THE STORE SIDE, and it is a separate wall + from `action_catalog(rt)`'s. Withholding a row from the MENU stops it being offered; it does + not stop a hand-written body naming the kind, and the picker is not a security boundary + [[opening-a-route-widens-every-field]]. ⚠ Keyword-only in effect and defaulting to None, so + every existing caller is untouched by construction — the same shape `notes` used, for the + reason this docstring already gives about a signature change failing at RUN, not at import. + ⛔ `rt=None` REFUSES a gated kind rather than allowing it (`TENANT_GATED_ACTIONS`). + + Ids are STABLE: a caller's well-formed `id` is kept, so selecting an action in the builder + survives a Save. A missing or colliding one is minted `act_`; minting on every clean would + move the selection under the person editing it. + + ⭐⭐ D-75 — `notes` IS THE DISCLOSURE CHANNEL, AND IT IS OPT-IN. Pass a list and this function + appends one plain sentence per thing it SILENTLY CHANGED: a `create_record` condition dropped + (see the note at that branch), and any config key an arm's allowlist did not keep. + + ⛔ WHY IT IS AN OUT-PARAMETER RATHER THAN A THIRD RETURN VALUE. D-75's own exit says *"this is + a signature change across its callers"* and that is exactly what makes the obvious fix + dangerous mid-wave: `clean_actions` is called from `clean_flow`, from `_is_untouched_ig_seed` + and recursively from its own group arm, and a caller that unpacked two values from a + three-tuple fails at RUN, not at import. An optional list defaults to `None`, every existing + caller is untouched by construction, and the one door that wants to tell somebody opts in. + ⚠ IT REPORTS, IT NEVER REFUSES. Every drop here is deliberate and D-65 is the reason — refusing + a stored automation's condition would 400 it forever, with no way to edit it out, because the + editor cannot save the automation it needs to fix. A drop is recoverable; a locked door is not. + What was missing was never the refusal, it was somebody being told. + """ + if raw in (None, ""): + return [], None + if not isinstance(raw, list): + return None, "actions must be a list" + seen = _seen if _seen is not None else set() + count = _count if _count is not None else [0] + out = [] + for entry in raw: + if not isinstance(entry, dict): + return None, "each action must be an object with a kind" + kind = _s(entry.get("kind"), 30).strip() + row = next((r for r in ACTION_CATALOG if r["kind"] == kind), None) + if row is None: + return None, (f"{kind or 'that action'!r} is not one of: " + + ", ".join(a["kind"] for a in ACTION_CATALOG)) + if not row["ready"]: + return None, (f"{row['label']!r} is on the menu but not built yet. " + f"{row['detail'][0].lower()}{row['detail'][1:]}") + # ⭐⭐ W35-T35 (C8) — THE TENANT WALL. Refused, not dropped, and this is the one place in + # this function where a refusal is right: D-65's "drop, never refuse" protects a stored + # automation from becoming permanently unsavable, and NO tenant this can refuse has ever + # been able to store one — the kind is withheld from their catalog, so there is no legacy + # body to strand. Dropping it silently would instead let a flow save, look saved, and never + # do the step the person configured. + if not _tenant_may_use(kind, rt): + return None, (f"{row['label']!r} is not available in this workspace. It sends as " + f"Royal Imports, using Royal Imports' own mail credentials") + count[0] += 1 + if count[0] > MAX_ACTIONS: + return None, f"an automation runs at most {MAX_ACTIONS} actions" + aid = _s(entry.get("id"), 40).strip() + if not re.fullmatch(r"act_[a-z0-9_]{1,32}", aid or "") or aid in seen: + n = 1 + while f"act_{n}" in seen: + n += 1 + aid = f"act_{n}" + seen.add(aid) + # ⭐ WAVE 26 · C4 / owner ruling R9 — A `create_record` CARRIES NO CONDITION, EVER. + # + # Owner, correcting their own earlier answer mid-grill: *"if it is Create Record, I don't + # think you can even add Conditions at all. That's not how the Create Record works."* + # Airtable agrees, and so does the shape: every other action operates ON the record the + # flow is walking, so "run this only when …" is a question about something + # that exists. Create record MAKES one. There is nothing to test yet, which is why the + # picker on this panel was empty and why every condition saved against it named no field — + # the owner's *"the condition on action act_1 names no field"*. The condition belongs on + # the trigger, or on a conditional group wrapping the action. + # + # ⛔ DROPPED, NOT REFUSED, AND THE DIFFERENCE IS D-65. Refusing would 400 every stored + # automation that already carries one — forever, with no way to edit it out, because the + # editor cannot save the automation it needs to fix. A drop is recoverable; a refusal is a + # locked door. + # ⚠ Dropped SILENTLY, and that is defensible only because the panel goes with it: wave 26 + # removes the condition editor from this action kind entirely (C4), so there is no control + # whose value could appear to be ignored. If a `create_record` condition editor ever comes + # back, this needs a disclosure channel — `clean_actions` has none today. + if kind == "create_record": + # D-75: the drop is unchanged; what is new is that a caller can be TOLD about it. + if notes is not None and entry.get("when"): + notes.append(f"{aid}: the condition on a Create-record step was dropped. A " + f"Create record has no record to test yet. Put the condition on the " + f"trigger, or on an If wrapping this step.") + when = None + else: + when, cerr = clean_cond(entry.get("when"), where=f"action {aid}") + if cerr: + return None, cerr + cfg_raw = entry.get("config") if isinstance(entry.get("config"), dict) else {} + cfg, cerr = _clean_action_config(kind, cfg_raw, depth, seen, count, notes=notes, rt=rt) + if cerr: + return None, cerr + # ⛔ D-75 — THE ALLOWLIST'S OWN DROPS, DIFFED HERE RATHER THAN REPORTED BY EACH ARM. Every + # arm builds a fresh `out = {...}`, so none of them knows what it did not keep; the caller + # of the arm does, because it holds both dicts. One diff, and a new arm is covered the day + # it is written rather than the day somebody remembers to instrument it. + if notes is not None and isinstance(cfg, dict): + gone = [k for k in cfg_raw if k not in cfg] + if gone: + notes.append(f"{aid}: {', '.join(sorted(gone))} " + f"{'is' if len(gone) == 1 else 'are'} not a setting a " + f"{KIND_LABELS.get(kind, kind)!r} step uses, so " + f"{'it was' if len(gone) == 1 else 'they were'} not saved.") + out.append({"id": aid, "kind": kind, + "enabled": bool(entry.get("enabled", True)), + "when": when, "config": cfg}) + return out, None + + +def _clean_action_config(kind, cfg, depth, seen, count, notes=None, rt=None): + """One action's `config`, per kind. Returns `(config, error)`. + + ⚠ `rt` is threaded ONLY so the `group` arm can hand it back to `clean_actions` for the branch + recursion (W35-T35 / C8). Without it a tenant-gated kind is refused at the top level and + ACCEPTED inside an If, which is the half of a flow `_walk_actions`' own docstring says people + cannot see. + """ + if kind == "group": + # ⭐ WAVE 24 · C-FORK (owner ruling R8) — a group is a FORK now: `{branches: [...]}`, + # each `{id, label, cond, actions}`. It was one condition with one action list, i.e. an + # if with no else, so expressing "otherwise" meant a second group carrying the negation + # by hand — two conditions a later edit could put out of step with each other. + if depth + 1 > MAX_GROUP_DEPTH: + return None, (f"conditional groups nest at most {MAX_GROUP_DEPTH} deep. " + f"past that a flow is a flowchart and belongs on the board") + raw_branches = cfg.get("branches") + if not isinstance(raw_branches, list) or not raw_branches: + # ⛔ MIGRATION, NOT A REFUSAL. The shipped shape is exactly one implicit branch, and + # the live automations carry it — refusing it would 400 every Save of a flow that + # was legal yesterday. Read-side, so nothing has to be rewritten in the store. + raw_branches = [{"id": "b1", "label": "A", "cond": cfg.get("cond"), + "actions": cfg.get("actions")}] + if len(raw_branches) > MAX_BRANCHES: + return None, (f"an If / then has at most {MAX_BRANCHES} branches. Past that the " + f"record is being routed, which is what the board's lanes are for") + out_branches, bseen = [], set() + for i, br in enumerate(raw_branches): + if not isinstance(br, dict): + return None, "each branch of an If / then is an object" + cond, cerr = clean_cond(br.get("cond"), where="the branch") + if cerr: + return None, cerr + if cond is None and i != len(raw_branches) - 1: + # ⛔ ONE Otherwise, and it is LAST. `_walk` takes the first branch that matches + # and a null condition matches everything, so a catch-all above another branch + # would make every branch below it dead code — silently, and only for the + # records that reached it. Refused with the reason rather than reordered: this + # module does not quietly rewrite what somebody built. + return None, ("only the LAST branch of an If / then can be the Otherwise leg. " + "a catch-all above another branch makes the ones below it " + "unreachable") + # D-75: threaded into the branch too — an unconfigured step inside an If is + # exactly the one a person cannot see, which is `_walk_actions`' own argument. + kids, kerr = clean_actions(br.get("actions"), depth + 1, seen, count, + notes=notes, rt=rt) + if kerr: + return None, kerr + if not kids: + return None, "a branch with no actions inside it does nothing" + bid = _s(br.get("id"), 40).strip() + if not re.fullmatch(r"b[a-z0-9_]{0,32}", bid or "") or bid in bseen: + bid = f"b{i + 1}" + bseen.add(bid) + label = " ".join(_s(br.get("label"), 40).split()) or ( + "Otherwise" if cond is None else _branch_letter(i)) + out_branches.append({"id": bid, "label": label, "cond": cond, "actions": kids}) + return {"branches": out_branches}, None + if kind in ("update_record", "create_record"): + values = cfg.get("values") + if not isinstance(values, dict) or not values: + return None, f"the {kind.replace('_', ' ')} action writes no values" + if len(values) > MAX_ACTION_VALUES: + return None, f"one action writes at most {MAX_ACTION_VALUES} cells" + clean_vals = {} + for k, v in values.items(): + key = _s(k, 80).strip() + if not key: + return None, "a value is written to a field with no name" + if v is not None and not isinstance(v, (str, int, float, bool)): + return None, (f"the value for {key!r} must be text or a number. " + f"an automation writes cells, not objects") + clean_vals[key] = _s(v, 500) if isinstance(v, str) else v + out = {"values": clean_vals} + # ⭐ WAVE 24 — an OPTIONAL self-given name, so a SEEDED action can say what it is + # ("Create an Instagram record", C-TRIG law 3) instead of wearing the catalog's generic + # "Create record". Follows the `review` arm below, which has carried `config.label` since + # wave 23 — one precedent, not a new mechanism, and it lives inside the untyped `config` + # bag so no shared client type has to change for it. + lbl = " ".join(_s(cfg.get("label"), 60).split()) + if lbl: + out["label"] = lbl + if kind == "create_record": + table = _s(cfg.get("table"), 60).strip() + if not table: + return None, "the create record action names no database" + if not table.startswith(UT_PREFIX): + return None, (f"actions write to blank databases (ut_*) this wave. " + f"{table!r} is not one") + out["table"] = table + # ⭐ WAVE 25 · C5 (owner ruling R1a) — UNIQUE ON. Without it this action APPENDS + # forever (`_commit_action_writes` mints `max+1`), so ANY scheduled flow carrying a + # Create record duplicates a row per run — silently, and worse every day. + # + # ⛔ `""` IS THE STORED DEFAULT AND IT MEANS TODAY'S APPEND. Not a migration, not a + # guess: a stored automation must not change meaning because this key arrived. The + # upsert is a thing somebody turns on, the same shape `clean_ending` gave `terminal`. + # + # ⛔ AND IT MUST NAME A FIELD THIS ACTION ACTUALLY WRITES. Upserting on a key the + # action never sets means every incoming row carries a BLANK key — `upsert_rows` + # counts those as `skipped` and writes nothing at all. That is a control that reads + # as "no duplicates" and delivers "no rows", which is the worse failure by a distance. + # Same shape as `scrape_db`'s "the key field must be one of the mapped fields" + # (`clean_config`, above) — one precedent, not a second mechanism. + unique = _s(cfg.get("uniqueOn"), 80).strip() + if unique and unique not in clean_vals: + return None, (f"this action does not write {unique!r}, so it cannot keep records " + f"unique on it. It writes: " + ", ".join(sorted(clean_vals))) + out["uniqueOn"] = unique + return out, None + if kind == "send_statement": + # ⭐⭐ WAVE 35 · T35 / R10 — WHAT A STATEMENTS STEP IS CONFIGURED WITH: a customer filter + # and a template. Both are the SENDER's own vocabulary (`routes_statements.send` passes + # `templates: {subject, intro, footer}` straight to `collections_send.queue_statement`), + # so this arm names no field the send door does not already take. + # + # ⚠ EVERY TEMPLATE FIELD IS OPTIONAL AND EMPTY MEANS "THE DEFAULT", which is exactly how + # the send door already reads them (`t.get("subject") or cs.DEFAULT_SUBJECT`). Storing our + # own copy of the default instead would freeze today's wording into every stored agent and + # silently stop tracking `collections_send`'s. + out = {} + tier = _s(cfg.get("tier"), 40).strip() + # ⛔ REFUSED, NOT COERCED, AND THE ANSWER LISTS THE REAL ONES — the same shape the connector + # cadence uses. A tier nobody sends to would filter the batch to zero customers and report + # a successful run: a silent no-op is the worst outcome available here, because the person + # believes their customers were invoiced. + # ⚠ `""` IS LEGAL and means EVERY tier. It is the stored default, so an agent saved before + # anybody picks a filter does not change meaning the day this key arrives (D-65's rule). + if tier and tier not in STATEMENT_TIERS: + return None, (f"{tier!r} is not a collection tier. Pick one of: " + + ", ".join(STATEMENT_TIERS) + ", or leave it blank for all of them") + out["tier"] = tier + for key, cap in (("subject", 200), ("intro", 4000), ("footer", 4000)): + out[key] = _s(cfg.get(key), cap) + lbl = " ".join(_s(cfg.get("label"), 60).split()) + if lbl: + out["label"] = lbl + return out, None + if kind in ENRICH_KINDS: + # ⭐ WAVE 30 · T08 — BOTH networks share this branch. See `ENRICH_KINDS`: the selection, the + # cooldown, the limit clamps and their reasons are network-agnostic, and a second copy for + # TikTok would drift until one network accepted a limit the other refused. + # ⭐ WAVE 25 · C4. The switches are `field_instagram`'s, unchanged in meaning — the paid + # rung, its fallback, the per-post engagement buy and the dry run — because they are the + # things that decide what a run COSTS and R4 moves the capability, not the controls. + # + # ⚠ `profileField` MAY BE EMPTY, and that is the A3 stored-inert pattern rather than + # laxness: the C3 profile FLAG resolves it at run time, so the action can be added to a + # flow before anybody has flagged the column. An unresolvable binding fails CLOSED with a + # sentence at run (`profile_field_key`), which is the honest half — refusing the save + # would make the action unaddable on a database that has not been flagged yet. + # C5: ONE validator, shared with `clean_config`'s machine-kind branch. Two independent + # clamps on the same key is how the action and the legacy kind come to disagree about + # what "10 posts" means. + # + # ⚠ `submitted=False` HERE, AND IT IS THE OPPOSITE OF THE OTHER CALL SITE. The difference + # is not the value, it is what "present" MEANS on each path. `clean_config` gets a PATCH, + # so a key being there is a person having typed it. This function gets the whole action + # list re-posted on every save (`_pin_unique`'s note says so in as many words), so a + # `maxPosts` here is usually just the client echoing back what was already stored — and + # every action written before this wave stored the old default of 24. Refusing would + # therefore 400 those automations forever on a number nobody chose, which is D-65. + # The ceiling is still taught, in the place where a person can act on it: the panel's + # input is bounded at MAX_POSTS_PER_PULL (C5 tells B not to offer more). + max_posts, perr = clean_max_posts(cfg.get("maxPosts"), submitted=False) + if perr: + return None, perr + # ⭐ 2026-08-07 (owner ruling) — THE SELECTION, and every key here is OPTIONAL with a + # working default. That is D-65's lesson applied ahead of time rather than after: an enrich + # action stored before this wave carries none of these, and it must keep saving and keep + # running. Absent `fromView` = the whole database; absent `limit` = DEFAULT_ENRICH_LIMIT; + # `skipRecent` absent = OFF, because a cooldown nobody asked for silently stops enriching. + # ⚠ `limit` is CLAMPED here and again in `enrich_selection`. Not belt-and-braces: this door + # sees a person's typed number and can refuse politely, while the selection sees whatever + # is STORED — including configs written before the cap existed. Clamping only here would + # let an old 5000 through at run time. + try: + e_limit = int(cfg.get("limit") or DEFAULT_ENRICH_LIMIT) + except (TypeError, ValueError): + return None, "how many records to enrich has to be a number" + if e_limit < 1: + return None, "an enrich step has to be allowed at least one record" + try: + e_days = int(cfg.get("skipRecentDays") or DEFAULT_ENRICH_COOLDOWN_DAYS) + except (TypeError, ValueError): + return None, "the number of days has to be a number" + groups, gerr = clean_post_groups(cfg.get("postGroups"), max_posts) + if gerr: + return None, gerr + return {"profileField": _s(cfg.get("profileField"), 80).strip(), + "fromView": _s(cfg.get("fromView"), 80).strip(), + "sortField": _s(cfg.get("sortField"), 60).strip() or DEFAULT_ENRICH_SORT, + # Anything that is not "asc" is newest-first, so a typo cannot invert the order a + # person is spending against — it lands on the documented default instead. + "sortDir": "asc" if str(cfg.get("sortDir") or "").strip().lower() == "asc" + else "desc", + "limit": min(e_limit, MAX_ENRICH_PER_RUN), + "skipRecent": bool(cfg.get("skipRecent")), + "skipRecentDays": max(1, e_days), + # ⛔ `tier`/`noFallback` are ACCEPTED AND IGNORED here for the same reason as in + # `clean_config` (R5 / C2): a stored action carrying them still saves and simply + # loses them, because refusing a retired key 400s every definition written before + # this wave (D-65). + # ⛔ THE FLAG THAT MULTIPLIES THE BILL BY THE POST COUNT. Off unless asked for. + # ⭐ WAVE 30 · T10 — THE TIKTOK FORCE-OFF IS GONE, because the capability it was + # standing in for now exists. T08 shipped `... and kind != "enrich_tiktok"` here + # deliberately: the key was ACCEPTED (never a 400, D-65's lesson) and answered + # honestly with `False`, because storing a `True` nothing acts on is a money switch + # claiming a capability nobody built. Both networks now read the SAME two keys. + "postMetrics": bool(cfg.get("postMetrics")), + # The separate Comments dataset is more granular than a post read and stays off + # until the user intentionally enables it. + "commentMetrics": bool(cfg.get("commentMetrics")), + "dryRun": bool(cfg.get("dryRun")), + # ⭐ 2026-08-09 (owner) — "last 12 reels / 12 videos", per GROUP. Absent = OFF, so + # every action stored before today keeps saving and keeps running unchanged. + "postGroups": groups, + "maxPosts": max_posts}, None + if kind == "find_records": + table = _s(cfg.get("table"), 60).strip() + if not table: + return None, "the find records action names no database" + cond, cerr = clean_cond(cfg.get("cond"), where="the find") + if cerr: + return None, cerr + try: + limit = int(cfg.get("limit") or 25) + except (TypeError, ValueError): + return None, "the find records limit must be a whole number" + if limit < 1 or limit > FIND_LIMIT_MAX: + return None, f"the find records limit is between 1 and {FIND_LIMIT_MAX}" + return {"table": table, "cond": cond, "limit": limit}, None + if kind == "ai_agent": + # ⭐⭐ W33-T56 — THE FUZZY STEP'S CONFIG. Same posture as the web arm below in every + # respect that matters: an ALLOWLIST, a shape test on the url, a clamped timeout, and an + # EMPTY config stored rather than refused so the step can be added before it is configured + # (the wave-23 illegal-default-seed defect, and D-79's scar). + # ⚠ IT DELIBERATELY DOES NOT TAKE A `selector`. Working the selector out is the step; a + # selector box here would be a control that contradicts the action's own reason to exist. + _u = _s(cfg.get("url"), 500).strip() + if _u and not (_u.startswith("http://") or _u.startswith("https://") + or _u.startswith("{{")): + return None, "a web address must start with http:// or https://" + try: + _t = int(cfg.get("timeoutMs") or WEB_READ_TIMEOUT_MS) + except (TypeError, ValueError): + return None, "the web step timeout must be a whole number of milliseconds" + try: + _max = int(cfg.get("maxSteps") or AI_AGENT_MAX_STEPS) + except (TypeError, ValueError): + return None, "the step ceiling must be a whole number" + out = {"url": _u, + # ⚠ LONGER THAN A SELECTOR AND SHORTER THAN A PROMPT. This is a sentence or two + # describing a job, and the model is given the page's own vocabulary at run time — + # a 4,000-character instruction is a sign somebody is writing a program in prose. + "instruction": _s(cfg.get("instruction"), 600), + "field": _s(cfg.get("field"), 80).strip(), + "timeoutMs": max(1000, min(_t, WEB_READ_TIMEOUT_MAX_MS)), + # ⛔ THE CEILING IS THE POINT OF THE CLAMP, not the default. Every step this action + # composes is a real browser action inside one job, and a fuzzy instruction is + # exactly the input that produces twenty of them. Bounded here so a run cannot be + # talked into an unbounded journey by a sentence. + "maxSteps": max(1, min(_max, AI_AGENT_MAX_STEPS))} + if cfg.get("dryRun"): + out["dryRun"] = True + return out, None + if kind in WEB_KINDS: + # ⭐⭐ WAVE 31 · T38 (C5 / R10) — AND THIS ARM IS THE HALF THAT WOULD HAVE SHIPPED MISSING. + # + # ⭐⭐ W31 QA WIDENED IT FROM `web_read` TO ALL FIVE KINDS, AND THAT IS THE SAME DEFECT THIS + # ARM'S OWN COMMENT DESCRIBES, ARRIVING A SECOND TIME. When the four write kinds flipped to + # `ready:True` on the owner's revocation of R5, they became storable — and with this arm + # still testing `kind == "web_read"` they fell through to `return {}, None` below, keeping + # their KIND and losing `url` / `selector` / `value` / `hint` **silently, with no error + # anywhere**. Addable, storable, runnable, and forever unconfigured. The paragraph below + # was written about exactly this, one kind earlier; widening the test is what makes it + # true of the feature rather than of one row in it. + # + # ⛔ FOUND BY THE GATE, NOT BY REVIEW, AND IT IS THE TICKET'S OWN FAILURE MODE. Flipping + # `ready:True` and mounting the dispatch arm is not enough: with no branch here the + # function falls through to `return {}, None` below, so a stored `web_read` action keeps + # its KIND and loses its `url`, `selector` and `field` **silently, with no error anywhere**. + # The action would be addable, storable, runnable — and would fetch `""` forever. That is + # precisely the *"addable and inert"* outcome E's hand-off warned about, arriving through a + # different door than the one being watched, and the same silent-drop class as `patch`'s + # hand-maintained key list one screen up. + # + # ⚠ THE URL IS NOT `guard`ed HERE, on purpose. `guard` does DNS, and this is a validator + # that runs on every save of every automation — a save must not block on name resolution, + # and a template like `https://{Domain}/p` is not resolvable until a record supplies the + # value. The rail is enforced where the fetch happens (the job, and `web_agent`'s own + # refusal), which is the only place the FINAL url exists. What is enforced here is the + # shape: http(s) or an interpolation that could become one. + # ⛔⛔ AN EMPTY CONFIG IS STORED, NOT REFUSED — THE A3 STORED-INERT PATTERN, and the first + # draft of this arm got it wrong in a way only a gate could see. Requiring `url` / + # `selector` / `field` here makes the action UNADDABLE: `AutomationBuilder::seedFor` mints + # a new step from a seed and persists it immediately, so a validator that refuses an empty + # seed yields a red banner and no card — the wave-23 illegal-default-seed defect, caught by + # `verify_automation_ui`'s catalog-derived seed check the moment `web_read` became ready. + # ⇒ The same posture the enrich arm already takes with `profileField`: a step may be added + # to a flow before it is configured, and it FAILS CLOSED AT RUN with a sentence naming what + # is missing. What is refused here is a value somebody actually TYPED and got wrong — never + # the absence of one they have not typed yet. + url = _s(cfg.get("url"), 500).strip() + # ⚠ `{{` IS THE TEMPLATE FORM — `interpolate`'s syntax is `{{field_key}}`, NOT `{field}`. + # A url that STARTS with a placeholder is legitimate (the record supplies the host), so the + # shape test admits it; anything else must be an absolute http(s) address. + if url and not (url.startswith("http://") or url.startswith("https://") + or url.startswith("{{")): + return None, "a web address must start with http:// or https://" + selector = _s(cfg.get("selector"), 200).strip() + field = _s(cfg.get("field"), 80).strip() + try: + timeout = int(cfg.get("timeoutMs") or WEB_READ_TIMEOUT_MS) + except (TypeError, ValueError): + return None, "the web step timeout must be a whole number of milliseconds" + timeout = max(1000, min(timeout, WEB_READ_TIMEOUT_MAX_MS)) + out = {"url": url, "selector": selector, "field": field, + "attr": _s(cfg.get("attr"), 40).strip() or "text", + "all": bool(cfg.get("all")), "timeoutMs": timeout} + wait = _s(cfg.get("waitFor"), 200).strip() + if wait: + out["waitFor"] = wait + # ⭐ THE FOUR WRITE KINDS' OWN KEYS. Carried for EVERY web kind rather than switched on + # `kind`, because the cost of carrying an unused key is nothing and the cost of dropping a + # used one is a step that runs forever against a value the user typed and cannot see. + # `value` is the text `web_fill` types; `secret` masks it in the run log; `hint` is the + # human label `web_repair` follows when a selector has gone stale; `dryRun` rehearses a + # write without performing it (E measured the field reading back empty afterwards). + value = _s(cfg.get("value"), 500) + if value: + out["value"] = value + hint = _s(cfg.get("hint"), 200).strip() + if hint: + out["hint"] = hint + if cfg.get("secret"): + out["secret"] = True + if cfg.get("dryRun"): + out["dryRun"] = True + return out, None + return {}, None + + +def _branch_letter(i): + """0 -> 'A', 1 -> 'B', … (R8's lettering). Past 'Z' it doubles rather than wrapping, so a + label is never reused — `MAX_BRANCHES` makes that unreachable, and a silent collision is + worse than an ugly name.""" + return chr(65 + i % 26) * (1 + i // 26) + + +def group_branches(act): + """The branches of a group action, MIGRATING the pre-wave-24 `{cond, actions}` shape. + + ⭐ ONE READER for the fork's shape, because the alternative is what this module keeps paying + for: `clean_actions`, `walk_actions` and the runner would each need to know that a stored + group might carry either shape, and the one that forgot would silently skip a live + automation's actions. A definition cleaned since wave 24 always carries `branches`; one read + straight out of the store (the runner reads definitions, not payloads) may not. + """ + cfg = (act or {}).get("config") or {} + brs = cfg.get("branches") + if isinstance(brs, list) and brs: + return [b for b in brs if isinstance(b, dict)] + return [{"id": "b1", "label": "A", "cond": cfg.get("cond"), + "actions": cfg.get("actions") or []}] + + +def walk_actions(actions): + """Every action in the tree, depth-first, groups included. One walker, so "how many actions + does this flow have" and "which review stages exist" cannot answer differently. + + ⚠ WAVE 24: a group's children live under its BRANCHES now. This walker feeds the action + COUNT, `review_actions` (hence the board's stages) and `compose_sentence` — so a version + that still read `config.actions` would report a fork's contents as zero actions, hide every + review inside a branch from the board, and let `MAX_ACTIONS` be exceeded without noticing. + """ + for act in actions or []: + yield act + if act.get("kind") == "group": + for br in group_branches(act): + for kid in walk_actions(br.get("actions")): + yield kid + + +def interpolate(text, row): + """`{{field_key}}` → the record's value. Unknown keys resolve to EMPTY, deliberately: an + action that wrote the literal `{{status}}` into a cell because somebody typo'd the key would + put template syntax in front of a customer, and a blank is the visible failure.""" + if not isinstance(text, str) or "{{" not in text: + return text + def _sub(m): + v = (row or {}).get(m.group(1).strip()) + return "" if v is None else str(v) + return re.sub(r"\{\{([^}]{1,80})\}\}", _sub, text) + + +# ───────────────────────────────────────────────────────────────────────────────────────────── +# WAVE 23 · C4/C5/C6 — THE ACTION RUNNER. One record at a time, actions in declared order. +# +# ⛔ ONE COALESCED WRITE PER TABLE PER RUN. The runner accumulates patches in memory and commits +# them once at the end (the 256-commits/hr budget every writer in this module respects). A +# per-action `rt.update` would be correct and would also spend the tenant's whole commit budget +# on one busy flow. +# ⛔ A REVIEW SUSPENDS THAT RECORD AND NOTHING ELSE. When a card reaches a review action the +# runner stamps its stage and stops walking THAT record — later actions belong to the branch a +# human (or the AI) has not chosen yet. Other records keep going; a gate is not a global pause. +# ⛔ EVERY WRITE HERE IS MACHINE-ORIGIN and goes through this module's own writers, never the +# human doors — which is exactly what keeps A2(2)'s loop prevention structural: an action's write +# cannot fire an event trigger, its own or a sibling's. + +def _act_row_patch(patches, table, rid, values): + patches.setdefault(table, {}).setdefault(str(rid), {}).update(values) + + +def profile_field_key(table, named="", source=PROFILE_SOURCE_IG): + """C3/C4: WHICH COLUMN on this database carries the profile handle. '' when none does. + + Resolution order, and the missing branch is the important one: + 1. the column the action NAMES — the person picked it, so it is not a guess; + 2. otherwise the column carrying C3's flag `profile: {source: "instagram"}`; + 3. otherwise **NOTHING**, and the run says so. + + ⛔ THERE IS DELIBERATELY NO FALL-BACK TO `_auto_url_field`. That helper finds "the first `url` + column" and is exactly right for `field_instagram`, whose whole configuration was a URL column + — but here it would silently bind the enrich step to whatever URL column happens to be first, + on a database where nobody has said which column is a profile. The failure would be a run that + reports success having enriched from the wrong column, which is worse than one that refuses: + a wrong number is harder to notice than a missing one, and it would be written into the + permanent history as if it had been measured. + + ⭐ WAVE 30 · T08 — `source` PARAMETERISES STEP 2 ONLY, defaulting to Instagram so every + existing caller is byte-for-byte unchanged. Step 1 (the column the person NAMED) is already + network-agnostic: a named column is a decision, not a guess, and second-guessing it against a + flag would refuse a binding somebody made on purpose. + ⛔ AND THE TWO SOURCES MUST NOT BE MERGED INTO "any profile flag". `ut_tt_profile.handle` + carries `profile: {source: "tiktok"}` and `ut_ig_profile.handle` carries `"instagram"`; a + resolver that accepted either would let a TikTok enrich step bind to an Instagram column and + spend money asking a TikTok dataset about an Instagram handle — a wrong number written into a + permanent history, which is the exact failure the no-fallback rule above exists to prevent. + """ + fields = (table or {}).get("fields") or [] + want = str(named or "").strip() + if want: + return want if any(str(f.get("key")) == want for f in fields) else "" + for f in fields: + p = f.get("profile") + if isinstance(p, dict) and str(p.get("source") or "") == str(source): + return str(f.get("key") or "") + return "" + + +def _flow_table(defn): + """The database a flow's actions and ending operate on: the automation's own target, or the + table its event trigger watches when the flow has no target of its own. + + ⭐ WAVE 30 · T05 — the discovery arm covers BOTH kinds. It read `== "discover_instagram"`, so a + TikTok discovery with no stored `targetTable` fell to the trigger's table — which a corpus + search does not have — and resolved to `""`. `_presets_after_write` asks this function where to + spawn the preset columns, so an empty answer there is a database that never appears. + """ + cfg = (defn or {}).get("config") or {} + if defn.get("kind") in DISCOVERY_KINDS: + return cfg.get("targetTable") or discovery_facts(defn.get("kind"))[1] + return cfg.get("targetTable") or ((defn.get("trigger") or {}).get("table") or "") + + +#: ⭐ 2026-08-07 (owner ruling) — WHAT ONE ENRICH STEP MAY SPEND IN A SINGLE RUN. +#: `limit` is the number the panel offers; this is the ceiling the module enforces whatever the +#: panel sends, and it is DISCLOSED when it bites rather than applied as a silent `[:N]` +#: ([[no-unverifiable-aggregates]]). 100 profiles is ~4 minutes of vendor pacing on its own. +MAX_ENRICH_PER_RUN = 100 +DEFAULT_ENRICH_LIMIT = 25 +DEFAULT_ENRICH_COOLDOWN_DAYS = 30 +#: Which column the selection orders by when nobody has said. `first_found` newest-first is the +#: owner's own example ("top N enrichment sorted by date") and the useful default: it enriches what +#: discovery just found rather than re-walking the oldest leads in the book. +DEFAULT_ENRICH_SORT = "first_found" + +#: ⭐⭐ 2026-08-09 (DEBT D-103) — WHERE A RUN'S PER-RECORD REASONS TRAVEL. +#: A run's `counts` are integers and `_commit_run` drops everything non-numeric, so the sentence +#: a vendor gave us had nowhere to ride and was thrown away at three separate layers. This key is +#: a deliberate passenger IN the counts dict, popped by `run_now` before the counts are stored. +#: ⛔ A RESERVED KEY, NOT A COUNT: it must never be rendered as one, which is why it starts with +#: an underscore — `COUNT_LABELS` on the client is an allow-list and cannot pick it up by +#: accident, and `_commit_run`'s numeric filter is the second net under it. +RUN_NOTES_KEY = "_notes" +#: ⭐⭐ WAVE 28 / OWNER RULING R9 — `NOT_FOUND_RETRY_DAYS` IS RETIRED. A handle a VENDOR SAID DOES +#: NOT EXIST is now a TOMBSTONE: never auto-retried, at any interval. +#: ⛔ THE 30-DAY BACKOFF WAS THE DEFENSIBLE ANSWER AND IT WAS STILL WRONG, which is the sentence +#: worth keeping. Its argument was that a handle can be renamed back or a suspension lifted, so +#: the verdict should expire. But nothing about US changes on day 31 — the only new information +#: is a guess that the world moved — so the timer buys one paid vendor call per dead handle per +#: month, forever, on a row nobody is looking at, and reports it as "1 blocked". A tenant with a +#: hundred stale handles pays a hundred times a month to be told the same thing. +#: ⭐ WHAT RE-ARMS IT IS A HUMAN, and there are two doors: +#: 1. the KEY is `(platform, handle)` — so correcting a typo re-arms IMMEDIATELY, with no +#: wiring at all, because the corrected handle simply is not the one we recorded; and +#: 2. `clear_gone()` — called when the profile CELL is written, so re-typing the SAME handle +#: (a person saying "try it again") also re-arms. +#: ⚠ The run keeps NAMING the skipped handles every time, not only on the run that discovered +#: them: a tombstone the owner cannot see is just a disappearance. +#: A queued profile snapshot the vendor never finishes is dropped after this, with a note. An +#: unbounded pending list is the forever-loop this whole change exists to remove, wearing the +#: opposite mask ([[gate-answers-the-wrong-question]]). +PENDING_PROFILE_MAX_HOURS = 24 + + +def _order_key(value): + """Order one CELL. Numbers numerically, everything else as text. + + ⚠ Cells are stored as STRINGS, so `"20872"` and `"9"` compare in the wrong order as text — + which on a `followers` sort is not a cosmetic wrong order, it is the wrong 25 accounts getting + paid for. Numbers are tried first and text is the fallback, never the other way round. + ⚠ Dates need no special case: R3 made these columns `date` and they store `YYYY-MM-DD`, where + lexicographic order IS chronological order. If that format ever changes, this comment is the + thing that was relied on. + """ + s = str(value if value is not None else "").strip() + try: + return (0, float(s.replace(",", "")), "") + except ValueError: + return (1, 0.0, s.lower()) + + +def _days_since(day, today=None): + """Whole days between a `YYYY-MM-DD` stamp and today, or None when it cannot be read. + + ⛔ None means UNKNOWN and every caller must treat it as "not measured", never as 0 or as a + large number — the cooldown below reads an unreadable stamp as "never enriched", which spends + money rather than skipping. That is the right way round: a skipped record is invisible, and a + stamp we cannot parse is our bug, not a reason to silently stop enriching somebody's data. + """ + import datetime as _dt + s = str(day or "").strip()[:10] + if not s: + return None + try: + d = _dt.date.fromisoformat(s) + except ValueError: + return None + return ((today or _dt.date.today()) - d).days + + +def _hours_since(stamp): + """Whole hours since a full ISO stamp, or None when it cannot be read. + + ⚠ ISO, NOT `YYYY-MM-DD` — `_days_since` above answers a question in DAYS about a `date` + column and truncates to 10 characters. A pending snapshot's age is measured in hours and its + stamp carries a time, so reusing that function would read every entry as "queued at + midnight". Same reason its None means UNKNOWN: an unreadable stamp must not age a paid, + outstanding snapshot out of the queue. + """ + import datetime as _dt + s = str(stamp or "").strip() + if not s: + return None + try: + t = _dt.datetime.fromisoformat(s.replace("Z", "+00:00")) + except ValueError: + return None + now = _dt.datetime.now(_dt.timezone.utc) + if t.tzinfo is None: + t = t.replace(tzinfo=_dt.timezone.utc) + return max(0, int((now - t).total_seconds() // 3600)) + + +def _gone_key(row, handle, platform=None): + """The identity a "this account does not exist" verdict is remembered under. + + `(platform, handle)` — wave 26 R4's dedup key, not the handle alone, because `@x` on + Instagram and `@x` on TikTok are two accounts. Blank platform means Instagram, which is what + every row on a preset profile table is and what `preset_cells` stamps. + + ⭐ WAVE 30 · T08 — `platform` OVERRIDES THE ROW, and the override is what makes the key honest + on a table the row cannot speak for. The fallback above reads the row's own `platform` cell, + which the TikTok runner writes — but only AFTER the first successful pull. A hand-typed row on + an arbitrary database that a TikTok step is bound to by NAME (`profile_field_key` step 1, + which is deliberately network-blind) carries no such cell, so a TikTok "this account does not + exist" verdict would have been filed under `instagram:`. A tombstone has no expiry + (W28/R9), so that would permanently suppress an Instagram read of a DIFFERENT person's + account with the same handle — silently, and without ever spending the money that would have + shown it was wrong. The caller knows which network it asked; the row does not. + """ + plat = str(platform or (row or {}).get("platform") or PLATFORM_INSTAGRAM).strip().lower() + return f"{plat}:{str(handle or '').strip().lstrip('@').lower()}" + + +def clear_gone(rt, table_key, handle, row=None): + """R9's re-arm door: forget the "this account does not exist" verdict for ONE handle. + + Returns the number of automations whose memory was changed — 0 is the ordinary answer and is + not an error, because most cell edits are not on a dead handle. + + ⭐⭐ WHY THIS IS A PUBLIC FUNCTION IN THE ENGINE RATHER THAN A LOOKUP IN THE ROW WRITER. + The verdict lives on the AUTOMATION (`state.enrichNotFound`), not on the row — it has to, + because it is a fact about what a run learned and paid for. But the thing that re-arms it is a + ROW event, and the row writer must not need to know the shape of an automation's state to + trigger it. So the seam is one call with the three things the writer already has, and every + walk of `all_definitions` stays on this side of the fence. + ⚠ IT MATCHES THE SAME WAY THE SKIP DOES. `_gone_key` is the one implementation of "which + handle is this", so a verdict can never be recorded under a key this cannot find + ([[one-evaluator-per-question]]). + + ⛔ WITHOUT ITS CALLER THIS IS INERT, AND THAT IS RECORDED RATHER THAN ASSUMED. R9's first + door — correcting a typo — works with no wiring at all, because the key IS the handle and a + different handle is simply not the one we recorded. This second door only opens when the row + write path calls it, which lives in `routes_tables.patch_row` (another session's fence). + Until that line lands, re-typing the SAME dead handle stays skipped + ([[flag-shipped-without-its-writer]] — named on purpose, so it is not discovered later). + """ + key = _gone_key(row or {}, handle) + if not str(handle or "").strip(): + return 0 + changed = 0 + for auto_id, defn in (all_definitions(rt) or {}).items(): + if str((defn.get("config") or {}).get("targetTable") or "") != str(table_key): + continue + known = dict(((defn.get("state") or {}).get("enrichNotFound")) or {}) + if key not in known: + continue + known.pop(key, None) + # ⚠ `or None` — an empty dict must clear the key rather than store `{}`, which is what + # every other state writer here does and what keeps a definition from growing a graveyard + # of empty maps. + set_state(rt, str(auto_id), {"enrichNotFound": known or None}) + changed += 1 + return changed + + +def _prime_enrich_batch(chosen, rows, profile_key, table, enrich, prefetch, + step=_no_step, log=print): + """Buy the selection's profiles in ONE vendor call per chunk. Returns how many resolved. + + ⭐⭐ 2026-08-09 — THE FIX FOR "WHY IS THIS ONE RECORD PER CALL". `bd_scrape` has always taken a + LIST of URLs and its own docstring names the failure mode ("…turning a 20-profile run into an + hour"); the only profile caller passed a single-element list from inside the per-record walk. + MEASURED on nurilab: a 25-record walk still running at 67 minutes, 25 billed snapshots, 25 + vendor emails. This runs once, where the whole chosen set and the row bodies are both in hand. + + ⛔ IT IS A FAST PATH AND MUST STAY ONE. Everything it fails to resolve is simply absent from + `prefetch`, and the per-record rung then behaves exactly as it does today — including its + corpus fallback and its Apify second opinion. A batch that cannot make the run WRONG is a + batch that can be shipped on a paid path; that is why the vendor call is wrapped and a failure + returns 0 instead of raising. + + ⛔ A DEFERRED CHUNK IS FANNED OUT PER DESTINATION, NOT PER SNAPSHOT. One snapshot id now + serves many records, so each pending task keeps its OWN `influencer`/`table`/`rowId` and they + share the id — `_pending_profile_tasks` is explicit that a profile cannot be resolved from the + handle alone ("two databases may both hold `@x`"). The handle is ALSO written into `prefetch` + behind `DEFERRED_MARK` so the walk does not buy the same snapshot a second time. + """ + want, dests, handles = [], {}, [] + for rid in (chosen or []): + row = rows.get(str(rid)) or {} + h = str(row.get(profile_key, "") or "").strip().lstrip("@").lower() + if not h or h in prefetch: + continue + want.append((h, str(rid))) + if h not in dests: + handles.append(h) + dests.setdefault(h, []).append(str(rid)) + if not handles: + return 0 + step(f"Reading {len(handles)} profile{'' if len(handles) == 1 else 's'} from the source") + deferrals = [] + try: + nodes, note = bd_profiles_batch(handles, deferred=deferrals) + except Exception as exc: # noqa: BLE001 — never fatal, see docstring + log(f"[aios-auto] profile batch failed, falling back to per-record reads: " + f"{type(exc).__name__}: {exc}") + return 0 + prefetch.update(nodes) + if note: + log(f"[aios-auto] profile batch note: {_s(note, 300)}") + for d in deferrals: + sid = str(d.get("snapshotId") or "") + if not sid: + continue + for u in (d.get("urls") or []): + h = str(ig_handle(str(u)) or "").strip().lstrip("@").lower() + if not h or h in nodes: + continue + prefetch[h] = {DEFERRED_MARK: sid} + for rid in dests.get(h, []): + enrich["pendingProfiles"].append( + {"snapshotId": sid, "datasetId": str(d.get("datasetId") or ""), + "urls": [str(u)], "kind": "profile", "influencer": h, + "table": table, "rowId": rid, "requestedAt": _iso()}) + return len(nodes) + + +def enrich_selection(rt, table_key, cfg, profile_key, today=None, gone=None, platform=None): + """⭐ 2026-08-07 (owner ruling) — WHICH records this enrich step spends on, in order. + + Returns `(ordered_row_ids, note)`. The note is the honest account of what the selection did + and is surfaced in the run summary; `""` means there is nothing worth saying. + + ⛔ **`limit` IS A QUOTA OF WORK DONE, NOT A WINDOW OF ROWS EXAMINED**, and that is the owner's + ruling in as many words: *"if a user choose to enrich 30 and from that sorted list of 30, 20 is + enriched last 30 days, then it goes to next list like this"*. So the walk CONTINUES past every + skipped record until 30 have actually been enriched or the list runs out. The naive reading — + take the top 30, then filter — would have billed for 10 and reported success, and the number in + the box would have meant something different every run depending on how much of the top of the + list happened to be fresh. A quota is predictable; a filtered window is not. + + The order of operations, each step narrowing the one above: + 1. the optional saved VIEW — resolved through `view_filter`, the SAME resolver `enters_view` + and `seed_rows` use. A view that has been deleted is a PROBLEM, never an empty tree: an + empty tree matches everything, so degrading would turn "enrich my shortlist" into "enrich + the entire database", at vendor prices. + 2. the SORT — the owner's "top N sorted by date", blanks always last in both directions + (a blank is unknown, not smallest). + 3. the COOLDOWN — skip anything enriched within N days, when the toggle is on. + 4. the QUOTA — stop at `limit`, itself clamped to `MAX_ENRICH_PER_RUN`. + + ⚠ A record with a BLANK handle is skipped and never counted against the quota — there is + nothing to enrich and it must not consume a slot somebody paid for. + """ + t = ut_get(rt, str(table_key or "")) + if t is None: + return [], f"{table_key!r} is not a database in this workspace" + rows = dict(t.get("rows") or {}) + notes = [] + + view_id = str(cfg.get("fromView") or "").strip() + if view_id: + tree, fields, problem = view_filter(rt, table_key, view_id) + if problem: + # ⛔ REFUSE, do not widen. See the docstring — this is the branch where a quiet + # fallback costs real money. + return [], f"{ENRICH_VIEW_UNREADABLE}. {problem}" + # ⭐⭐ 2026-08-07 (owner report) — `filter_eval.matches`, NOT `lane_match`. THIS LINE WAS + # A THIRD IMPLEMENTATION OF "does this row match", AND IT SPOKE THE WRONG LANGUAGE. + # + # `view_filter` returns a SAVED VIEW's filter tree — `{"nodes": [{colId, op, value}], + # "conj"}` in the GRID's dialect, whose operators are `contains/eq/neq/gt/gte/lt/lte/ + # isEmpty/isNotEmpty/between/within`. `lane_match` reads an AUTOMATION LANE condition — + # `{field, op, value}` with `=/!=/>/includes/is_empty/…` — and dispatches on + # `COND_GROUP_KEYS`. Handed a view tree it found no group key, read `raw.get("field")`, + # got `""`, and answered **False for every row**: MEASURED on the owner's own automation, + # a view matching exactly one record selected NOTHING, reported NO error, and the run + # committed `ok`. *"I just chose the View 'Enrichment test' where there is only 1 manual + # record... how come it says 51 records walked?"* + # + # ⛔ AND THE RIGHT EVALUATOR WAS ALREADY IN THE FILE. `_row_gate`'s `enters_view` branch + # resolves the SAME tree through `harness.filter_eval.matches(tree, row, fields)` and has + # always been correct. `_seed_event_state`'s own docstring states the law this line broke: + # *"Two implementations of 'does this row match' is how a seed disagrees with the edge it + # is supposed to arm."* There were three. Now there are two callers of one function. + # ⚠ `fields` is no longer discarded — `matches` needs the column TYPES to compare a date + # as a date and a number as a number, which is the half `lane_match` could not have had. + import harness.filter_eval as filter_eval + rows = {rid: r for rid, r in rows.items() if filter_eval.matches(tree, r, fields)} + + sort_field = str(cfg.get("sortField") or DEFAULT_ENRICH_SORT).strip() or DEFAULT_ENRICH_SORT + newest_first = str(cfg.get("sortDir") or "desc").strip().lower() != "asc" + # ⭐⭐ 2026-08-07 (owner ruling) — A MISSING DATE MEANS **NOW**, NOT "UNKNOWN". + # + # Owner: *"treat a missing first found as now, and manual entry should go first, instead of + # treated as last. I want to see my inayma manual entry works with automation enrichment."* + # And they are right about the semantics, not just the preference: on these tables the ONLY + # rows without a `first_found` are ones a PERSON typed, because the discovery runner stamps it + # on every row it writes. So a blank is not missing data — it is a row that was first found + # today, by the person sitting in front of it, and the one they most want enriched. + # + # The old rule sorted blanks LAST in both directions, so a hand-typed handle sat at position + # 41 of 41 and fell outside a limit of 25 — the row the whole feature exists for was the one + # it never reached. + # + # ⛔ DATE COLUMNS ONLY, and the narrowing is the honest half. "Blank means now" is a fact about + # a TIMESTAMP; a blank `followers` is not "the most followers", it is genuinely unknown, and + # treating it as the maximum would spend the budget on the rows we know least about. So a + # non-date sort keeps the old rule: unknown sorts last, in both directions. + # + # ⚠ NOTE THIS FALLS OUT OF THE KEY RATHER THAN BEING A SECOND PASS — `(0, value)` for a real + # date and `(1, "")` for a blank, with `reverse` doing the rest. Newest-first puts blanks at + # the front (they are "now"); oldest-first puts them at the back (they are still "now"). One + # rule, both directions, no branch that can disagree with itself. + ftype = str(((next((f for f in (t.get("fields") or []) + if str(f.get("key")) == sort_field), None)) or {}).get("type") or "") + if ftype == "date": + ordered = sorted(rows.items(), + key=lambda x: (0, str(x[1].get(sort_field) or "").strip()) + if str(x[1].get(sort_field) or "").strip() else (1, ""), + reverse=newest_first) + else: + have = [(rid, r) for rid, r in rows.items() if str(r.get(sort_field) or "").strip()] + blank = [(rid, r) for rid, r in rows.items() if not str(r.get(sort_field) or "").strip()] + have.sort(key=lambda x: _order_key(x[1].get(sort_field)), reverse=newest_first) + blank.sort(key=lambda x: _rid_num(x[0])) + ordered = have + blank + + try: + quota = int(cfg.get("limit") or DEFAULT_ENRICH_LIMIT) + except (TypeError, ValueError): + quota = DEFAULT_ENRICH_LIMIT + quota = max(1, min(quota, MAX_ENRICH_PER_RUN)) + if cfg.get("limit") and quota != int(cfg.get("limit") or 0): + notes.append(f"the limit was capped at {MAX_ENRICH_PER_RUN} for one run") + + cooling = bool(cfg.get("skipRecent")) + try: + days = int(cfg.get("skipRecentDays") or DEFAULT_ENRICH_COOLDOWN_DAYS) + except (TypeError, ValueError): + days = DEFAULT_ENRICH_COOLDOWN_DAYS + days = max(1, days) + + picked, cooled, blank_handle = [], 0, 0 + # ⭐⭐ 2026-08-09 — HANDLES A VENDOR HAS ALREADY SAID DO NOT EXIST. + # + # ⛔ THE DEFECT THIS CLOSES IS STRUCTURAL, AND IT IS THE WORD "AGAIN" IN THE OWNER'S REPORT. + # A blocked read writes NO cells, so `enriched_at` stays unset, so `_days_since(None)` is + # None, so the cooldown above can never exclude the row — while `Followers is empty` keeps it + # in the Pending cohort by construction. MEASURED on nurilab: one dead handle, re-bought at + # 06:00 on three consecutive days, reported each time as an opaque "1 blocked". No vendor fix + # removes that loop; only a memory of the verdict does. + # + # ⚠ THEY ARE **NAMED**, NOT SILENTLY DROPPED. The whole point is that the owner can act — the + # note goes into the run every single time, not only on the run that discovered it, because a + # run that quietly reports "0 records walked" tomorrow puts them straight back at "wtf". + skipped_gone = [] + for rid, r in ordered: + if len(picked) >= quota: + break + raw_handle = str(r.get(profile_key) or "").strip() + if not raw_handle: + blank_handle += 1 + continue + # ⛔ R9 — NO EXPIRY. There is deliberately no date arithmetic here any more: a verdict is + # a verdict until a human edits the cell. An `at` stamp is still STORED (it is what the + # owner reads to know when we last paid to be told this), it is simply not a clock. + # ⚠ WAVE 30 · T08 — the SAME `platform` the runner will file a new verdict under, so the + # skip and the write cannot disagree about which account a tombstone belongs to. + if isinstance((gone or {}).get(_gone_key(r, raw_handle, platform)), dict): + skipped_gone.append(raw_handle) + continue + if cooling: + since = _days_since(r.get("enriched_at"), today=today) + if since is not None and since < days: + cooled += 1 + continue + picked.append(str(rid)) + + # ⛔ THE HONEST ACCOUNT. "10 enriched" reads as success whether the quota was 10 or 30, so the + # run says when it could NOT fill the quota and why — the same disclosure rule `cap_note` and + # `run_plain` follow. Silence here would make a shrinking selection invisible. + if cooled: + notes.append(f"{cooled} skipped as enriched in the last {days} days") + if skipped_gone: + shown = ", ".join(f"@{h}" for h in skipped_gone[:5]) + # ⚠ THE SENTENCE IS THE FEATURE. It must name the handles AND the two things a person can + # do, because nothing else will ever retry them — under R9 this note is the only path + # back from a tombstone, so a vaguer version would strand the row permanently. + notes.append(f"{len(skipped_gone)} skipped because Instagram has no such account " + f"({shown}{', …' if len(skipped_gone) > 5 else ''}). Delete the row or " + f"correct the handle; they are not retried automatically") + if blank_handle: + notes.append(f"{blank_handle} skipped with no handle") + if len(picked) < quota and (cooled or blank_handle or skipped_gone or ordered): + notes.append(f"{len(picked)} of the {quota} asked for. The list ran out") + return picked, "; ".join(notes) + + +def _has_action(actions, kind): + """Does this flow contain `kind` ANYWHERE, forks included? + + ⛔ FORKS ARE THE WHOLE REASON THIS IS A FUNCTION. A group's children live under + `config.branches[].actions` (C-FORK), so a flat scan of the top level answers False for an + enrich step somebody put inside an If — and the caller would then skip a schema top-up the + run genuinely needs. Same walk `mapTree` does on the client, and the same trap wave 24 + recorded when four hand-rolled walks all forgot to descend. + """ + for a in (actions or []): + if not isinstance(a, dict): + continue + if a.get("kind") == kind: + return True + for br in ((a.get("config") or {}).get("branches") or []): + if _has_action((br or {}).get("actions") or [], kind): + return True + return False + + +def _actions_of_kind(actions, kind): + """Every action of `kind`, including actions nested inside If branches.""" + out = [] + for action in actions or []: + if not isinstance(action, dict): + continue + if action.get("kind") == kind: + out.append(action) + for branch in ((action.get("config") or {}).get("branches") or []): + out.extend(_actions_of_kind((branch or {}).get("actions") or [], kind)) + return out + + +def _web_agent(): + """C5's seam, resolved LAZILY — `web_agent.run_step(step, ctx) -> (dict|None, str)`. + + ⚠ IMPORTED INSIDE THE CALL, like every `connectors_tt` site in this module, and here it also + buys a failure mode worth having: if `web_agent.py` is ever missing from a deployment, the + engine still imports and every OTHER action still runs — the web step alone reports a sentence. + A module-level import would turn one absent file into a dead automation module. + + ⛔ AND THE ABSENCE IS REPORTED, NEVER SWALLOWED (R6's second sentence). The shim below answers + the same `(None, sentence)` contract the real seam does, so the caller's `if why:` branch is + the only branch there has ever been. + """ + try: + import web_agent + return web_agent + except Exception as exc: # noqa: BLE001 + class _Absent: + @staticmethod + def run_step(_step, _ctx=None): + return None, ("The web-browsing agent is not available in this deployment " + f"({type(exc).__name__}). Nothing was read.") + return _Absent + + +def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print, + step=_no_step): + """Walk `flow.actions` for each of `row_ids`, then apply the ENDING. Returns the run counts. + + ⚠ The ending runs even when there are NO actions, and that is not a detail: a discovery + automation's review stage comes from `stages_for`, not from actions, + so an early return on an empty action list would have silently disabled the owner's loop + feature on the flagship flow — the one kind most likely to want `auto_reset`. Caught in + review; the tests all happened to pass a non-empty action list, which is why it was green. + """ + defn, _retired = _without_retired_board(defn) + flow = (defn or {}).get("flow") or {} + actions = flow.get("actions") or [] + # ⛔ THE PINNED STEP IS A PICTURE OF THE ENGINE'S OWN WRITE, NOT A SECOND ONE. + # + # MEASURED 2026-08-06: a discovery run wrote its candidates through `ut_write_rows` (the + # "Save results" node) and THEN walked the flow over the rows it had just written — where + # the seeded `create_record` inserted every one of them AGAIN. Two rows became four, on + # every run, silently, since wave 24 seeded that action. Making step 1 permanent (owner + # ruling, same day) would have made the duplication permanent and unavoidable with it. + # + # ⚠ THE CARD STAYS. It is what the owner asked for and it is honest — "this search puts what + # it finds in a database" is exactly what the engine does. What it must not do is do it + # twice. Skipped HERE rather than not-seeded, so the builder still shows the step and the + # server still refuses to let it be removed. + if ig_action_pinned(defn, 0) and actions: + actions = actions[1:] + # ⭐ `webRead` / `webBlocked` JOIN HERE (W31-T38, C5): declared up front like every other + # counter, so a run that read nothing and a run with no web step are DIFFERENT numbers. R6's + # second sentence is a reporting rule, and a counter that only exists once it is non-zero + # cannot report a refusal. + counts = {"actionsRun": 0, "updated": 0, "created": 0, "found": 0, + "webRead": 0, "webBlocked": 0} + #: W31-T38 — the once-per-run markers the web arm uses, so an unconfigured step says so ONCE + #: rather than once per record. Same shape as the enrich accumulators' `notes`. + web_notes = [] + # ⭐ C4 — THE ENRICH ACCUMULATOR. Every append row the enrich action produces is collected + # across the WHOLE walk and written once at the end, for the reason the module header states + # and `run_field_instagram` learned the hard way: `upsert_rows` rebuilds the row dict on + # entry, so calling it per record is O(existing) per record — survivable at 5,000 rows and an + # automation that never finishes at `MAX_UT_IG_ROWS`. One flush, three upserts. + enrich = {"snaps": [], "posts": [], "psnaps": [], "comments": [], "pending": [], + # ⭐ 2026-08-09 — deferred PROFILE snapshots (paid, still building at the vendor) + # and handles a vendor has said do not exist. Both are collected across the whole + # walk and written once, for the same reason every other list here is. + "pendingProfiles": [], "gone": {}, + "profiles": 0, "ok": 0, "blocked": 0, "notes": [], "dry": False} + # ⭐⭐ WAVE 30 · T08 — THE TIKTOK ACCUMULATOR IS ITS OWN, and the separation is the design + # decision rather than a convenience. `_enrich_flush` is not network-agnostic: it calls + # `ensure_ig_graph`, addresses the four `IG_*_TABLE` constants and writes through to the + # platform Instagram master (`ig_master.append_run`). Routing TikTok rows through it would + # append a TikTok creator's followers into Instagram's pooled history, which the metric + # FIELDS read (W29-T34) — a wrong number in a permanent series, i.e. the same failure class + # as `preset_cells`' unconditional `PLATFORM_INSTAGRAM` stamp that `tt_preset_cells` exists + # to avoid. + # ⚠ AND THE KEYS IT FLUSHES ARE PREFIXED `tt…` FOR A MEASURED REASON: `apply_actions` merges + # both flushes into ONE `counts` dict, so a shared `enriched`/`enrichBlocked` key would let a + # flow carrying BOTH an Instagram and a TikTok step report one network's numbers under the + # other's name, last writer winning, with no error anywhere. + # ⭐ W30 · D-156 — `pending` is the TikTok twin of Instagram's, and it is a RUN-level list for + # the same reason theirs is: a media snapshot belongs to a HANDLE, not to a row, so nothing + # about it needs the per-record sink that `pendingProfiles` needs. + tt = {"snaps": [], "posts": [], "psnaps": [], "comments": [], "pending": [], + "profiles": 0, "ok": 0, "blocked": 0, "notes": [], "dry": False} + # The verdicts this automation already holds, read ONCE — `enrich_selection` consults them + # per record and re-reading the definition per row would be a store read per record. + known_gone = dict(((defn or {}).get("state") or {}).get("enrichNotFound") or {}) + # ⭐ 2026-08-07 — THE SELECTION, RESOLVED ONCE PER ACTION AND NOT ONCE PER RECORD. + # `enrich_selection` sorts and scans the whole table; doing that inside the per-record walk + # would be O(rows²) and, worse, would re-answer "which 25 records" every time a record walked + # through — so the quota could never be honoured. Keyed by ACTION id, because two enrich steps + # in one flow are two independent budgets. + enrich_plan = {} + # ⭐⭐ 2026-08-09 — `{handle: node}` FOR THE WHOLE SELECTION, BOUGHT IN ONE CALL PER CHUNK. + # The vendor read used to happen inside the per-record walk, one URL per `/v3/scrape` and a + # `PACE_SECONDS` floor between records, so a 25-record walk was 25 billed snapshots and was + # MEASURED still running at 67 minutes. `bd_scrape` always accepted a list; nothing ever + # handed it one. Run-scoped rather than per-action: two enrich steps over the same table are + # two budgets (`enrich_plan`) but the same profiles, and a handle already bought is not worth + # buying twice. + enrich_prefetch = {} + if not table_key: + return counts + table = str(table_key) + # ⭐⭐ 2026-08-07 (owner report) — THE PRESET COLUMNS MUST EXIST BEFORE THE CELLS ARE WRITTEN. + # + # ⛔ MEASURED on the owner's own row: a successful enrich reported `enriched: 1` and filled + # NINE cells out of a pull that carried far more, because `_act_row_patch` can only write a + # cell whose COLUMN is declared — `user_tables` filters an unknown key on every write door. + # So every preset the target table did not already happen to have was silently dropped, and + # the run still said ok. Owner: *"UNTIL I SEE THAT ALL OF INAYMA'S INSTAGRAM FIELDS GET + # FILLED"* — this is the half that was swallowing them. + # + # ⚠ WHY IT WAS MISSING RATHER THAN BROKEN: the columns are spawned by `_presets_after_write`, + # which runs when a **Create record** action is SAVED (W25/R10). A `plain` automation whose + # only step is `enrich_instagram` never saves one, so nothing had ever declared them — the + # feature worked on discovery tables purely because discovery creates them for its own reasons. + # + # ⚠ ONCE PER RUN, not once per record, and only when an enrich step is actually present: + # `ut_ensure` MERGES by key and returns without a write when nothing is new, so the steady + # state is one dict comparison. Guarded by the same walk that would use it, so a flow with no + # enrich step never touches the schema of the database it walks. + # ⛔ ONLY ON A TABLE THAT IS ALREADY BOUND, and the narrowing is the careful half. The preset + # set declares `handle` with `pinned` AND `profile: {source: 'instagram'}` — so topping up an + # ARBITRARY database would (a) reshape someone's schema with 36 columns because they pointed a + # step at it, and (b) add a SECOND profile column to a table that already flagged a different + # one, which `user_tables` forbids at both write doors. A table with no profile column is an + # UNBOUND enrich: it must stay unbound and say so, which is a different defect (D-79) with its + # own honest refusal, not something to paper over by inventing the binding. + _tbl_now = ut_get(rt, table) or {} + _enrich_actions = _actions_of_kind(actions, "enrich_instagram") + _bound = next((f for f in (_tbl_now.get("fields") or []) + if isinstance(f.get("profile"), dict)), None) + if _bound is None: + _named = next((str((a.get("config") or {}).get("profileField") or "").strip() + for a in _enrich_actions + if str((a.get("config") or {}).get("profileField") or "").strip()), "") + _bound = next((f for f in (_tbl_now.get("fields") or []) + if str(f.get("key") or "") == _named), None) + if _bound and _enrich_actions: + # ⚠ AND THE DECLARATIONS ARE STRIPPED FROM ANYTHING NEW. The binding already exists — + # `_bound` is it — so a preset arriving now must carry DATA, never a second identity: a + # table whose profile column is `ig_handle` would otherwise gain a rival `handle`. + topup = [({k: v for k, v in f.items() if k not in ("profile", "pinned")} + if f.get("key") != _bound.get("key") else dict(f)) + for f in PRESET_PROFILE_FIELDS] + try: + if any(not bool((a.get("config") or {}).get("dryRun")) for a in _enrich_actions): + ensure_ig_graph(rt, username, str(defn.get("id") or ""), + profile_table=table, profile_field=str(_bound.get("key") or "")) + else: + # Dry run keeps its original no-create contract; only the already-established + # Profile table is topped up, and no canonical related database is spawned. + ut_ensure(rt, _tbl_now.get("label") or table, topup, username, key=table, + flow_tag=str(defn.get("id") or ""), lock_fields=True) + except Exception as exc: # noqa: BLE001 + # A schema top-up that cannot run must not stop the enrichment: the cells that DO + # have columns still land, which is strictly better than the pull being thrown away. + log(f"[aios-auto] preset top-up on {table} failed: {type(exc).__name__}: {exc}") + # ⭐⭐ WAVE 30 · T08 — THE SAME TOP-UP FOR TIKTOK, AND THE SAME NARROWING: only on a table that + # is ALREADY BOUND. Separate from the block above because every constant in it is Instagram's + # (`PRESET_PROFILE_FIELDS`, `ensure_ig_graph`) and because the binding is resolved by a + # DIFFERENT question — `profile_field_key(..., source=PROFILE_SOURCE_TT)`, so a TikTok step + # cannot adopt an Instagram column and go on to ask a TikTok dataset about an Instagram handle. + # ⚠ It runs on a dry run too, exactly as the Instagram branch does: the target's own columns + # are topped up, and the related append database is NOT spawned (that is the flush's job, and + # it returns before writing anything on a dry run). + _tt_actions = _actions_of_kind(actions, "enrich_tiktok") + if _tt_actions: + _tt_named = next((str((a.get("config") or {}).get("profileField") or "").strip() + for a in _tt_actions + if str((a.get("config") or {}).get("profileField") or "").strip()), "") + _tt_bound = profile_field_key(_tbl_now, _tt_named, source=PROFILE_SOURCE_TT) + if _tt_bound: + try: + ut_ensure(rt, _tbl_now.get("label") or table, + _tt_profile_schema_for(_tt_bound), username, key=table, + flow_tag=str(defn.get("id") or ""), lock_fields=True) + except Exception as exc: # noqa: BLE001 + log(f"[aios-auto] TikTok preset top-up on {table} failed: " + f"{type(exc).__name__}: {exc}") + rows = dict((ut_get(rt, table) or {}).get("rows") or {}) + if not rows: + return counts + # ⛔ NOTHING HERE TRACKS WHERE A RECORD "GOT TO" ANY MORE (wave 27 item 12, ruling R3). The + # walk used to keep a `reached` map and stamp each record with the furthest action it made — + # a board column's worth of bookkeeping written into the tenant's own table on every run. + # The board is deleted, so the stamp had no reader, and a machine column nobody reads is a + # cell the customer has to look at and a store commit we pay for on every scheduled run. + # The RUN LOG is the operational record of what happened now, and it is the only one. + patches, creates = {}, {} + + #: ⭐⭐ W33-T58 · D-191 — this record's web results, keyed by ACTION ID, and the run's job + #: count. `web_done` is cleared per record (a batch is built from one row's interpolated + #: values and means nothing for the next one); `web_jobs` counts JOBS across the whole run, + #: which is what `MAX_WEB_JOBS_PER_RUN` has always been trying to bound. + #: ⚠ A key present with `None` means "this step was in a batch that refused" — distinct from + #: absent, which means "no batch has covered it yet". Collapsing the two would re-run the + #: whole batch once per step of it, which is the defect this fixes, inverted. + web_done: dict = {} + web_jobs = [0] + + def _web_step_dict(act, row): + """One action + one record → the step dict the seam takes. Interpolated HERE, as before. + + ⚠ `interpolate` on the CALLER's side, exactly as the update/create arms do it, so + `{{Field}}` works in a url, a selector or a typed value. The seam takes a plain dict and + does not know about rows. + """ + cfg = act.get("config") or {} + wait = cfg.get("waitFor") + step = {"kind": act.get("kind"), "id": str(act.get("id") or ""), + "url": interpolate(str(cfg.get("url") or ""), row), + "selector": interpolate(str(cfg.get("selector") or ""), row), + "attr": str(cfg.get("attr") or "text"), + "all": bool(cfg.get("all")), + "waitFor": interpolate(str(wait), row) if wait else None, + "timeoutMs": int(cfg.get("timeoutMs") or 20000)} + # ⚠ ADDED ONLY WHEN PRESENT: the seam distinguishes a key that is absent from one that is + # empty, and an empty `value` on a `web_read` would be a typed blank. + if cfg.get("value"): + step["value"] = interpolate(str(cfg.get("value")), row) + if cfg.get("hint"): + step["hint"] = interpolate(str(cfg.get("hint")), row) + if cfg.get("secret"): + step["secret"] = True + if cfg.get("dryRun"): + step["dryRun"] = True + return step + + def _run_web_batch(acts, start, row, rid): + """Run the longest safe run of consecutive web steps from `acts[start]` in ONE job. + + ⛔ **CONSECUTIVE, AND ONLY WHILE NOTHING IN THE BATCH DEPENDS ON THE BATCH.** `run_plan` + sends every step to one browser at once, so a step whose url/selector/value interpolates a + column an EARLIER step in the same batch writes would be interpolated against the value + that column had BEFORE the batch ran. That is a wrong answer rather than a slow one, so the + batch is cut immediately before any such step and the remainder becomes the next batch. + The single-step case is then exactly the old behaviour, which is what makes this safe to + land on a live flow. + ⚠ A step that is disabled, filtered out by its `when`, or unconfigured ENDS the batch + rather than being skipped inside it: each of those is a reason this record does not run + that step, and the loop's own arms already report them one at a time with their own + sentences. Ending here keeps exactly one place that decides what a blocked step says. + """ + batch, produced = [], set() + for act in acts[start:]: + kind = act.get("kind") + if kind not in WEB_KINDS or not act.get("enabled", True): + break + if not lane_match(act.get("when"), row): + break + cfg = act.get("config") or {} + if _web_missing(kind, cfg): + break + # The dependency cut. `interpolate` reads `{{Name}}`; a step naming a column an + # earlier step in THIS batch writes has to wait for the next job. + refs = " ".join(str(cfg.get(k) or "") for k in ("url", "selector", "value", "hint")) + if any(("{{" + f) in refs or ("{{ " + f) in refs for f in produced): + break + batch.append(act) + if str(cfg.get("field") or ""): + produced.add(str(cfg.get("field"))) + if len(batch) >= MAX_WEB_STEPS_PER_JOB: + break + if not batch: + return + steps = [_web_step_dict(a, row) for a in batch] + # ⛔ THE SEAM REFUSES A JOURNEY WHOSE FIRST STEP CARRIES NO ADDRESS — there is no page to + # act on yet — and a refusal is for the WHOLE plan. Batching a url-less first step would + # therefore take its followers down with it, where one-job-per-step only lost that step. + # A batch that cannot start is cut to one, which is exactly the old behaviour. + if not str(steps[0].get("url") or "").strip(): + batch, steps = batch[:1], steps[:1] + web_jobs[0] += 1 + + def _block(a, why_one): + web_done[str(a.get("id") or "")] = None + counts["webBlocked"] += 1 + if why_one: + log(f"[aios-auto] {a.get('kind')}: {why_one}") + + # ⛔ THE SEAM'S OWN BOUNDARY IS ON `run_step`, NOT ON `run_plan` — `run_step` wraps its + # call in `try/except` and calls that "the LAST boundary". Calling `run_plan` directly + # steps around it, and this code runs inside a record walk where an escaping exception + # ends the whole run. So the boundary moves here with the call. + try: + rows_out, why = _web_agent().run_plan( + steps, {"tenant": str(defn.get("tenant") or ""), + "automationId": str(defn.get("id") or ""), + "runId": str(defn.get("id") or ""), "log": log}) + except Exception as exc: # noqa: BLE001 — the LAST boundary + why, rows_out = (f"The web steps failed unexpectedly ({type(exc).__name__}: " + f"{str(exc).splitlines()[0][:200]}). Nothing was read."), None + if why: + log(f"[aios-auto] web: {why}") + for a in batch: + _block(a, "") + return + # ⚠ MATCHED BY ID, NEVER BY POSITION. The job returns a row for a step that FAILED and for + # every step after it that was never attempted, so the list can be shorter than, or + # misaligned with, the plan — and a positional read would hand step 3's caller step 2's + # answer, writing a wrong value into a real column, which is worse than the missing one it + # replaced. `_clean_step` mints an id for every step and the runner echoes it back, so the + # id is carried the whole way and is the only thing worth matching on. + by_id = {str(r.get("id") or ""): r for r in (rows_out or []) if isinstance(r, dict)} + for a in batch: + hit = by_id.get(str(a.get("id") or "")) + # ⛔ `ok` IS CHECKED HERE BECAUSE `run_step` USED TO CHECK IT. It turned a row with + # `ok:false` into a sentence and returned no result; reading `hit` without that test + # would take a failed step's empty `value` and write it over a real cell. + if not hit or not hit.get("ok"): + _block(a, str((hit or {}).get("error") or "") + or "The browser job returned nothing for this step.") + continue + web_done[str(a.get("id") or "")] = hit + + def _walk(acts, row, rid, depth=0): + """Walk the actions in order for ONE record. + + ⚠ THE RETURN VALUE IS VESTIGIAL and is deliberately kept as `False`. It used to mean + "this record was SUSPENDED by a review gate" — the one branch that could stop a walk + early. With review retired nothing suspends anything, so every record walks its whole + flow; the signature stays so a future gate-style action has an obvious place to say so. + """ + for idx, act in enumerate(acts): + if not act.get("enabled", True): + continue + if not lane_match(act.get("when"), row): + continue + kind = act.get("kind") + cfg = act.get("config") or {} + counts["actionsRun"] += 1 + if kind == "group": + # ⭐ WAVE 24 · C-FORK — FIRST MATCHING BRANCH WINS, and only that one runs. + # Declaration order is priority order, the same rule `route_record` applies to + # the board's lanes, and the Otherwise leg (`cond: null`) is simply the branch + # nothing above it beat — `lane_match(None, row)` is True, and `clean_actions` + # has already guaranteed a null condition can only be LAST. + for br in group_branches(act): + if lane_match(br.get("cond"), row): + if _walk(br.get("actions") or [], row, rid, depth + 1): + return True + break + elif kind == "update_record": + vals = {k: interpolate(v, row) for k, v in (cfg.get("values") or {}).items()} + row.update(vals) # later actions see the write, as they must + _act_row_patch(patches, table, rid, vals) + counts["updated"] += 1 + elif kind == "create_record": + target = str(cfg.get("table") or "") + vals = {k: interpolate(v, row) for k, v in (cfg.get("values") or {}).items()} + # C5: keyed by (table, uniqueOn) rather than by table alone, because two actions + # may legitimately write to ONE database on different keys — collapsing them onto + # the table would silently apply one action's uniqueness rule to the other's rows. + creates.setdefault((target, str(cfg.get("uniqueOn") or "")), []).append(vals) + counts["created"] += 1 + elif kind == "send_statement": + # ⭐⭐ WAVE 35 · T35 / R10 — THE ARM LANDS WITH THE CATALOG ROW, ON PURPOSE. + # + # `_walk` has no terminal `else` (see the note on `enrich_tiktok` in the catalog): + # an unknown kind is walked, COUNTED, reports the run `ok` and writes nothing. So a + # catalog row whose arm arrives in a later ticket is addable, storable and silently + # inert — a step a person configured, that reports success and does nothing. This + # arm exists so that window never opens. + # + # ⛔ T35 DOES NOT SEND AND DOES NOT PARK. R10's review stage is W35-T36; until it + # lands this says so out loud and counts the record as blocked, which is the same + # shape `ai_agent` uses for a step it cannot perform. It must never fall through to + # "ok". + # ⛔ AND IT NEVER SENDS FROM HERE, in this wave or any later one. The send door is + # `routes_statements`, behind SAFE_MODE + `admin_gate` + the tenant gate, reached by + # a human click on the review batch. This arm's whole job is to PREPARE. + if "send_statement_pending" not in web_notes: + web_notes.append("send_statement_pending") + log("[aios-auto] send_statement: statements are assembled for review, not " + "sent. The review batch is not configured yet, so nothing was prepared " + "and nothing was sent.") + counts["webBlocked"] += 1 + continue + elif kind == "odoo_sync": + # Same refusal, same reason as the arm below: a catalog row with no arm is walked, + # counted, and reports success having done nothing. The connector owns this sync. + if "odoo_sync_not_run_here" not in web_notes: + web_notes.append("odoo_sync_not_run_here") + log("[aios-auto] odoo_sync: the Odoo pull runs on the connector's own " + "schedule, not from this canvas. Nothing was done") + counts["webBlocked"] += 1 + continue + elif kind == "ai_enrich": + # ⛔⛔ A REFUSAL ARM, AND IT EXISTS FOR THE REASON THE `send_statement` ARM ABOVE + # STATES: `_walk` has no terminal `else`, so a catalog row whose arm is missing is + # walked, COUNTED, reports the run `ok` and writes nothing — a step somebody + # configured that succeeds at doing nothing. Adding the row (D-277) without this + # arm would have opened exactly that window. + # ⚠ AND THE REFUSAL IS THE TRUTH, not a stub. An AI column is filled by + # `ai_enrich`, driven from the column's own editor; the synthetic agent row this + # kind appears in is DERIVED from that column and is never stored, so nothing + # reaches here through the ordinary path. If something ever does, it must say so + # rather than report success. + if "ai_enrich_not_run_here" not in web_notes: + web_notes.append("ai_enrich_not_run_here") + log("[aios-auto] ai_enrich: an AI column is filled from the column's own " + "editor, not from this canvas. Nothing was done") + counts["webBlocked"] += 1 + continue + elif kind == "ai_agent": + # ⭐⭐ W33-T56 (owner item 7, ruling R3) — THE FUZZY STEP, AT RUN TIME. + # + # A description becomes concrete web steps HERE, against this record's own values, + # and then rides the ordinary seam. Composing at run time rather than at save time + # is the whole point: `{{Website}}` is a different page for every row, so a journey + # fixed at save time would be the same guess repeated. + # ⛔ IT COMPOSES ONLY `web_*` KINDS. The composer is handed the same catalog the + # AI-agent module uses, filtered to what a browser job can perform — so a fuzzy + # instruction cannot talk this action into writing a record or calling a connector. + # The blast radius of a bad sentence is one browser session, not the tenant. + # ⚠ AND IT REPORTS THE STEPS IT ACTUALLY TOOK, which is the ticket's own + # `done-when`. A step that composes a journey and reports only its final value is + # unauditable: nobody can tell a right answer from a lucky one. + _missing = _web_missing(kind, cfg) + if _missing: + _note = f"{kind}_unconfigured" + if _note not in web_notes: + web_notes.append(_note) + log(f"[aios-auto] {kind}: this step still needs " + + ", ".join(_missing) + ". Nothing was done") + counts["webBlocked"] += 1 + continue + if web_jobs[0] >= MAX_WEB_JOBS_PER_RUN: + if "web_cap" not in web_notes: + web_notes.append("web_cap") + log(f"[aios-auto] web: this run stopped after {MAX_WEB_JOBS_PER_RUN} " + f"browser jobs of about 10-30 seconds each.") + counts["webBlocked"] += 1 + continue + # W35 · C7: `st` + `user` so the model spend is attributed (`NOTE E-16`). + _plan, _why = _ai_agent_plan(cfg, row, log, st=rt, + user=str(defn.get("createdBy") or "")) + if _why: + # ⛔ NAMED, NEVER OPAQUE — the second half of the `done-when`. "The assistant + # could not work out how to do that" with the reason attached is actionable; + # a blank cell is not. + log(f"[aios-auto] ai_agent: {_why}") + counts["webBlocked"] += 1 + continue + web_jobs[0] += 1 + try: + _rows_out, _why2 = _web_agent().run_plan( + _plan, {"tenant": str(defn.get("tenant") or ""), + "automationId": str(defn.get("id") or ""), + "runId": str(defn.get("id") or ""), "log": log}) + except Exception as _exc: # noqa: BLE001 — the LAST boundary + _rows_out, _why2 = None, ( + f"the browser job failed unexpectedly ({type(_exc).__name__}: " + f"{str(_exc).splitlines()[0][:200]}). Nothing was done.") + if _why2: + log(f"[aios-auto] ai_agent: {_why2}") + counts["webBlocked"] += 1 + continue + # THE ACCOUNT OF WHAT IT DID — one line per step, in order, with each step's own + # verdict. This is what makes a fuzzy step auditable at all. + _done = [r for r in (_rows_out or []) if isinstance(r, dict)] + for _i, _r in enumerate(_done, 1): + log(f"[aios-auto] ai_agent step {_i}/{len(_plan)}: {_r.get('kind')} " + f"{'ok' if _r.get('ok') else 'FAILED'}" + + (f". {str(_r.get('error'))[:160]}" if not _r.get("ok") else "")) + _last = _done[-1] if _done else {} + if not _done or not _last.get("ok"): + log("[aios-auto] ai_agent: the journey did not finish. " + + str((_last or {}).get("error") + or "the browser job returned nothing for the last step")) + counts["webBlocked"] += 1 + continue # ⛔ NOTHING IS WRITTEN on an unfinished journey. + _target = str(cfg.get("field") or "") + if _target: + vals = {_target: _last.get("value")} + row.update(vals) + _act_row_patch(patches, table, rid, vals) + counts["webRead"] += len(_done) + elif kind in WEB_KINDS: + # ⭐⭐ WAVE 31 · T38 (C5) — THE WEB-BROWSING AGENT'S LIVE ARM, ALL FIVE KINDS. + # + # ⛔ WHY THIS ARM EXISTS SEPARATELY FROM THE RUNNER: session E built + # `web_agent.run_step` and **could not verify its own mounting**. An unmounted + # runner is a whole, correct, unreachable feature — the exact class five wave-29 + # features shipped as — so the mount and its `verify_wiring` row are C's, in one + # change. + # ⭐⭐ W31 QA WIDENED THIS ARM FROM `web_read` TO ALL FIVE KINDS on the owner's + # revocation of D-51/R5 (`TICKETS.md:1418`). ⚠ IT NEEDS NO PER-KIND HANDLING, and + # that is E's design rather than an omission: the runner normalises EVERY kind to + # set `result["value"]` (read → the text · goto → the title · click → the title it + # landed on · fill → the typed value, masked when secret · repair → the proposed + # selector), precisely so this one arm does not grow a switch that would be a + # second copy of the runner's table living in another lane's file. + # + # ⛔ IT BLOCKS FOR ~9-32 s (E measured it; `proto/web-agent-job.md` §4). That is + # tolerable HERE and nowhere else: this is the automation RUNNER, already a + # background walk. It must never be called from a route a person is waiting on. + # + # ⚠ `interpolate` ON THE CALLER'S SIDE, exactly as the update/create arms do it, so + # `{Field}` works in a url or a selector. E's seam takes a plain dict and does not + # know about rows. + # ⛔ FAIL CLOSED, ONCE, WITH THE REASON — the counterpart to the validator storing + # an unconfigured step (see `_clean_action_config`'s `web_read` arm). Reported once + # per RUN and not per record: the missing config is a property of the flow, so a + # 100-record walk would otherwise print the same sentence a hundred times and bury + # everything else. The same shape the enrich arm's `unbound` note uses. + # ⚠ PER KIND, because they do not need the same things: `web_goto` needs a url and + # no selector; `web_fill` needs a value nobody else takes; only `web_read` needs a + # column to write into. One shared three-field test would have blocked every + # `web_goto` ever configured for want of a selector it does not use. + _missing = _web_missing(kind, cfg) + if _missing: + # ⚠ THE NOTE KEY CARRIES THE KIND. It used to be the literal + # `"web_read_unconfigured"`, so a flow with an unconfigured `web_goto` AND an + # unconfigured `web_fill` would have reported the first and swallowed the + # second — once-per-RUN is the property, not once-per-FLOW. + _note = f"{kind}_unconfigured" + if _note not in web_notes: + web_notes.append(_note) + log(f"[aios-auto] {kind}: this step still needs " + + ", ".join(_missing) + ". Nothing was done") + counts["webBlocked"] += 1 + continue + # ⛔⛔ THE PER-RUN JOB CEILING (E-4). A browser job is ~9-32 s against HF's + # 6-concurrent cap — a flow over a few thousand rows would submit a few thousand + # jobs and run for days. Reported ONCE with its cause AND the fix, which is R6's + # second sentence: a limit that cannot be removed today must say why and what would + # remove it. `MAX_WEB_JOBS_PER_RUN` carries the reasoning. + # ⭐⭐ W33-T58 (D-191) — THE CEILING NOW COUNTS **JOBS**, NOT PAGES, because the + # two stopped being the same thing on the line below. A record whose three web + # steps batch into one job spends ONE of these, not three. + if web_jobs[0] >= MAX_WEB_JOBS_PER_RUN: + if "web_cap" not in web_notes: + web_notes.append("web_cap") + log(f"[aios-auto] web_read: this run stopped after " + f"{MAX_WEB_JOBS_PER_RUN} browser jobs of about 10-30 seconds each, " + f"which do not run in parallel. A record's consecutive web steps " + f"already share ONE job; to read more, narrow the flow's records.") + counts["webBlocked"] += 1 + continue + # ⭐⭐ W33-T58 · D-191 — ONE JOB FOR A RECORD'S CONSECUTIVE WEB STEPS. + # `run_plan(steps, ctx)` has always taken a list and nothing ever called it with + # more than one: the arm called `run_step`, which wraps `[step]`, so a flow with + # three web reads paid THREE ~9 s cold starts to do what one job does. The batch is + # built at WALK time rather than from the stored flow, because whether a step runs + # at all depends on this record (`enabled`, `when`, and whether it is configured). + if str(act.get("id") or "") not in web_done: + _run_web_batch(acts, idx, row, rid) + result = web_done.get(str(act.get("id") or "")) + if result is None: + continue # its batch refused; the reason was logged once, there + _target = str(cfg.get("field") or "") + if _target: + vals = {_target: (result or {}).get("value")} + row.update(vals) # later actions see the write, as they must + _act_row_patch(patches, table, rid, vals) + counts["webRead"] += 1 + elif kind == "enrich_instagram": + # ⭐ C4 (R3/R4). Reuses `pull_profile` and `capture_rows` — the SAME functions + # `run_field_instagram` calls, not a second implementation of either. + pkey = profile_field_key(ut_get(rt, table), cfg.get("profileField")) + if not pkey: + # FAIL CLOSED, ONCE, WITH THE REASON. Not per record: the binding is a + # property of the flow, so a 100-record run would otherwise put the same + # sentence in the log a hundred times and bury everything else. + if "unbound" not in enrich["notes"]: + enrich["notes"].append("unbound") + log("[aios-auto] enrich_instagram: no profile column on " + f"{table}. Name one on the action, or mark a text column as an " + "Instagram profile") + continue + # ⭐ 2026-08-07 (owner ruling) — IS THIS RECORD IN THIS RUN'S BUDGET? + # Resolved once (see `enrich_plan`) and then a membership test. A record outside + # the selection is NOT an error and NOT a skip worth logging per row — it is simply + # not this run's work, and the selection's own note already accounts for it. + aid = str(act.get("id") or "") + if aid not in enrich_plan: + chosen, sel_note = enrich_selection(rt, table, cfg, pkey, gone=known_gone) + enrich_plan[aid] = set(chosen) + if sel_note: + enrich["notes"].append(sel_note) + # ⭐ ONE VENDOR CALL PER CHUNK FOR THE WHOLE SELECTION, here and nowhere else: + # this is the only place the full chosen set and the row bodies are both in + # hand. Anything it resolves the per-record rung below reads from memory. + _prime_enrich_batch(chosen, rows, pkey, table, enrich, + enrich_prefetch, step, log) + if str(rid) not in enrich_plan[aid]: + continue + handle_raw = str(row.get(pkey, "") or "").strip() + if not handle_raw: + continue # nothing to enrich on this record, not an error + # ⚠ THE PACE FLOOR IS THE VENDOR'S, SO IT IS PAID ONLY WHEN THE VENDOR IS CALLED. + # A handle already in `enrich_prefetch` was bought by the batch above and is read + # from memory; sleeping 2.5 s before a dictionary lookup would hand most of the + # batching win straight back (25 records = ~62 s of pure waiting). + if enrich["profiles"] and str(handle_raw).strip().lstrip("@").lower() \ + not in enrich_prefetch: + time.sleep(PACE_SECONDS) # >=2 s between profiles (R7), as the runner does + enrich["profiles"] += 1 + # ⭐ THE DEFERRED-PROFILE SINK IS PER RECORD so the snapshot can be stamped with + # the row it belongs to: the collector writes preset cells back onto THAT record, + # and a run-wide list would have no way to say which handle each snapshot was for. + pend_prof = [] + # ⭐ W31-T39(c) — the DEFERRAL WATERMARK, taken before the pull. See the `partial` + # report below: a capability that was deferred is not a capability that failed, + # and `pull_profile` appends into these two sinks from inside the call. + _pend_before = len(enrich["pending"]) + res = pull_profile(handle_raw, + max_posts=cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL, + log=log, + post_metrics=bool(cfg.get("postMetrics")), + comment_metrics=bool(cfg.get("commentMetrics")), + pending_metrics=(enrich["pending"] + if not cfg.get("dryRun") else None), + pending_profile=(pend_prof + if not cfg.get("dryRun") else None), + prefetch=enrich_prefetch, + post_groups=cfg.get("postGroups")) + # ⭐ 2026-08-09 (owner: *"a Filter on our enrichment so we only get Last 12 reels / + # 12 videos etc … not just last 12 but by group also"*). `config.postGroups` is + # `[{"type": "video", "limit": 12}, …]`; a type nobody names is dropped. + # ⛔ DEFAULT OFF — with no `postGroups` the list is returned unchanged, so this + # costs nothing and changes nothing until somebody configures it. + # ⭐⭐ WAVE 30 · T16 — THE WINDOW IS NOW WIDENED BEFORE IT IS FILTERED, and the + # paragraph that used to sit here explaining why it could not be is gone with the + # reason. It said: *"it FILTERS the window we already bought … the views top-up + # runs INSIDE `pull_profile` against the posts it captured, so swapping a wider + # list in afterwards would hand back posts with no view counts."* Correct, and + # exactly why the widening went INTO `pull_profile` (`post_groups=`, above) rather + # than being bolted on out here — ahead of the top-up, not after it. + # ⚠ THIS LINE STAYS, and it is not redundant. A single-type ask is served by a + # native route that returns only that type; a MIXED ask has no such route, so the + # wide window comes back mixed and this is what keeps N of each. + if cfg.get("postGroups"): + res = {**res, "posts": select_post_groups(res.get("posts") or [], + cfg.get("postGroups"))} + for _p in pend_prof: + _p["table"], _p["rowId"], _p["requestedAt"] = table, str(rid), _iso() + enrich["pendingProfiles"].extend(pend_prof) + pulled = _iso() + if res["state"] in ("ok", "partial"): + enrich["ok"] += 1 + snap_row, idents, metric_rows, comment_rows = capture_rows(res, pulled) + enrich["snaps"].append(snap_row) + enrich["posts"].extend(idents) + enrich["psnaps"].extend(metric_rows) + enrich["comments"].extend(comment_rows) + # R3: LATEST onto the record itself. `row.update` too, so a LATER action in + # this same flow sees the enriched values — the rule `update_record` already + # follows, and without it "enrich, then route by follower count" would judge + # the record on the values it had before the pull. + # ⭐⭐ WAVE 31 · T39(c) — D-177(c): THE FIX APPLIED TO ONE PLATFORM AND NOT ITS + # TWIN, and that is the shape worth naming rather than the line. + # + # W30-T11 found this on TikTok the expensive way: a paid run enriched two + # profiles with `postMetrics` ON, wrote no posts, and said NOTHING — because + # `partial` takes THIS branch and only the `else` ever appended a note, so the + # reason `pull_profile` carries in `res["note"]` was discarded. Instagram has + # the identical branch and never got the fix; the sibling arm below has carried + # it since wave 30. One defect, two networks, one of them repaired — which is + # exactly how the last four TikTok/Instagram divergences were found. + # + # ⛔ ONLY WHEN IT WAS ASKED FOR, and ⛔ NOT WHEN THE BATCH DEFERRED — both + # conditions copied deliberately from the TikTok arm rather than re-reasoned. + # With `postMetrics` off, a `partial` is the normal answer for every profile + # and reporting it would put one useless line per record in the run entry; a + # DEFERRAL is a paid snapshot already filed for collection, and reporting it as + # a miss teaches a person to re-run and buy it twice. + # ⚠ Instagram's deferral sinks are `enrich["pending"]` (post metrics) and + # `pend_prof` (the profile snapshot) — the two lists `pull_profile` appends + # into — where TikTok reads `res["deferredMedia"]`. Different signal, same + # question: did this record's work get filed rather than lost? + if (cfg.get("postMetrics") and not (res.get("posts") or []) + and len(enrich["pending"]) == _pend_before and not pend_prof): + _why = _s(res.get("note"), 300) + if _why: + enrich["notes"].append(f"@{handle_raw}: {_why}") + cells = preset_cells(res, pulled) + if cfg.get("dryRun"): + enrich["dry"] = True + else: + row.update(cells) + _act_row_patch(patches, table, rid, cells) + else: + enrich["blocked"] += 1 + # ⭐⭐ D-103 — THE NOTE IS NAMED AND KEPT AT FULL LENGTH. + # It used to be `_s(note, 90)` with no handle attached: three runs blocked + # the same record and the store could not say which record, let alone why. + # 90 characters also truncated the one measured sentence mid-clause. This is + # the most valuable thing the run produces and it now reaches the run entry. + enrich["notes"].append( + f"@{handle_raw}: {_s(res.get('note'), 300) or res['state']}") + # A vendor STATING that the account does not exist is remembered, so the next + # run stops paying to be told the same thing. + if res.get("gone"): + enrich["gone"][_gone_key(row, handle_raw)] = { + "at": _iso(), "handle": handle_raw, + "note": _s(res.get("note"), 300)} + elif kind == "enrich_tiktok": + # ⭐⭐ WAVE 30 · T08 (carrying wave-29's dropped T05). THE SECOND NETWORK, and it + # is a sibling of the branch above rather than a flag inside it. What is genuinely + # shared is shared by CALL — `enrich_selection`, `enrich_plan`, the pace floor, + # the tombstone memory; what differs is the four things B-9 measured as + # Instagram-hardcoded, and each has a TikTok counterpart of its own name. + pkey = profile_field_key(ut_get(rt, table), cfg.get("profileField"), + source=PROFILE_SOURCE_TT) + if not pkey: + # D-79, on the network that did not exist when D-79 was written: FAIL CLOSED, + # ONCE, WITH THE REASON — and the marker is its OWN, so a flow carrying both + # steps cannot report the Instagram sentence about the TikTok one. + if "unbound" not in tt["notes"]: + tt["notes"].append("unbound") + log("[aios-auto] enrich_tiktok: no profile column on " + f"{table}. Name one on the action, or mark a text column as a " + "TikTok profile") + continue + aid = str(act.get("id") or "") + if aid not in enrich_plan: + # ⚠ ONE plan dict for both networks is correct and not an oversight: it is + # keyed by ACTION id, and an action has exactly one kind. Two steps over one + # table are two budgets whichever networks they read. + chosen, sel_note = enrich_selection(rt, table, cfg, pkey, gone=known_gone, + platform=PLATFORM_TIKTOK) + enrich_plan[aid] = set(chosen) + if sel_note: + tt["notes"].append(sel_note) + if str(rid) not in enrich_plan[aid]: + continue + handle_raw = str(row.get(pkey, "") or "").strip() + if not handle_raw: + continue # nothing to enrich on this record, not an error + # ⚠ THE PACE FLOOR IS THE VENDOR'S. There is no batch prefetch on this path yet + # (`pull_profile_tt` accepts one and nothing writes it — see the mailbox), so + # every profile after the first pays it. + if tt["profiles"]: + time.sleep(PACE_SECONDS) + tt["profiles"] += 1 + import connectors_tt as _tt_conn # lazy: connectors_tt imports this module + pend_prof = [] + res = _tt_conn.pull_profile_tt( + handle_raw, log=log, + pending_profile=(pend_prof if not cfg.get("dryRun") else None), + # ⭐ W30-T10. Both INCLUDE axes default OFF (W28/R5-R7), and the same + # `config` keys the Instagram step reads — one vocabulary, two networks. + max_posts=int(cfg.get("maxPosts") or 0), + post_metrics=bool(cfg.get("postMetrics")), + comment_metrics=bool(cfg.get("commentMetrics"))) + for _p in pend_prof: + _p["table"], _p["rowId"], _p["requestedAt"] = table, str(rid), _iso() + # The deferred-profile queue is the Instagram one BY DESIGN: it is a vendor + # snapshot id waiting to be collected, and `_pending_profile_tasks` keys tasks by + # table+row, not by network. Sharing it is what makes the tick finish a TikTok + # read it has already been charged for. + enrich["pendingProfiles"].extend(pend_prof) + # ⭐⭐ W30 · D-156 — A DEFERRED MEDIA BATCH IS FILED, NOT NARRATED. Before this, + # a posts or comments scrape the vendor took too long over was reported in the + # note and collected by NOTHING: paid for, and recoverable only by a human reading + # a sentence. `connectors_tt` has already filtered these to the media corpora, so + # a profile snapshot cannot arrive here; what the engine adds is the vocabulary the + # QUEUE speaks — the `kind` (from the dataset id, which is one-to-one on TikTok) + # and the HANDLE, which the transport never knew. + # ⚠ `dryRun` queues NOTHING: a dry run buys nothing, so an entry here could only + # be a fixture leaking, and filing it would make the tick collect a snapshot that + # was never paid for. + if not cfg.get("dryRun"): + for _d in (res.get("deferredMedia") or []): + _kind = tt_metric_kind(_d.get("datasetId")) + if not _kind: + continue # not a media corpus ⇒ not this queue's business + tt["pending"].append({**_d, "kind": _kind, "requestedAt": _iso(), + "influencer": str(handle_raw).strip() + .lstrip("@").lower()}) + pulled = _iso() + if res["state"] in ("ok", "partial"): + tt["ok"] += 1 + snap_row = tt_snapshot_row(res, pulled) + if snap_row: + tt["snaps"].append(snap_row) + tt_idents, tt_metrics, tt_comments = tt_capture_rows(res, pulled) + tt["posts"].extend(tt_idents) + tt["psnaps"].extend(tt_metrics) + tt["comments"].extend(tt_comments) + # ⭐⭐ WAVE 30 · T11 — A CAPABILITY THAT WAS ASKED FOR AND DID NOT ARRIVE IS + # REPORTED. This is R6's second sentence applied to a capability rather than to + # a row cap, and it was found the expensive way: the 09:14 UTC paid run on + # nurilab enriched two profiles with `postMetrics` ON, wrote no posts, and said + # NOTHING — `ok: true`, no note, no count — because `partial` takes the branch + # ABOVE and only the `else` ever appended a note. Every `partial` return in + # `pull_profile_tt` carries the reason in `note` (*"this account's row carried + # no post links"*, *"the post source returned nothing"*, or the vendor's own + # words), and all of them were being discarded. A person then sees two enriched + # rows, an empty posts database and no explanation anywhere. + # ⛔ ONLY WHEN IT WAS ASKED FOR, which is the difference between a report and + # noise: with `postMetrics` off, `pull_profile_tt` returns `partial` + *"post + # capture is off for this step"* for EVERY profile, and appending that would put + # one useless line per record into the run entry and let the summary quote it. + # ⛔ AND NOT WHEN THE BATCH DEFERRED — a deferral is not a failure to deliver, + # it is a paid snapshot already filed for collection + # (`ttEnrichMetricBatchesPending`, D-156), and reporting it as a miss would + # teach a person to re-run and buy it twice. + if (cfg.get("postMetrics") and not (res.get("posts") or []) + and not (res.get("deferredMedia") or [])): + _why = _s(res.get("note"), 300) + if _why: + tt["notes"].append(f"@{handle_raw} (TikTok): {_why}") + cells = tt_preset_cells(res, pulled) + if cfg.get("dryRun"): + tt["dry"] = True + else: + row.update(cells) + _act_row_patch(patches, table, rid, cells) + else: + tt["blocked"] += 1 + # ⚠ TAGGED, because `run_notes` now carries BOTH networks' per-record reasons + # (the two flushes are concatenated). The summary picks ONE note to quote, so an + # untagged TikTok line could be quoted under Instagram's sentence and vice versa + # — which would undo the whole point of giving each network its own sentence. + tt["notes"].append( + f"@{handle_raw} (TikTok): {_s(res.get('note'), 300) or res['state']}") + # ⛔ AND THERE IS DELIBERATELY NO TOMBSTONE WRITER HERE, which is the opposite + # of an oversight. On the Instagram side `res["gone"]` has exactly ONE source — + # Apify answering `ACCOUNT_GONE_NOTE` (`connectors_ig`); Bright Data has no + # not-found verdict at all, and `DEFAULT_CHAINS["tt_profile"]` is deliberately + # single-provider, so nothing on this chain can say "no account exists". A + # phrase match invented against unmeasured vendor output would file a + # PERMANENT verdict (W28/R9 — no expiry) on the strength of a guess. + # ⚠ The READ side is still network-scoped above (`platform=PLATFORM_TIKTOK`) + # and that half is load-bearing today: `known_gone` is shared, Apify DOES + # write `instagram:` verdicts, and without the scoping one of those + # would silently suppress a TikTok read of a different person's account. + elif kind == "find_records": + found = find_records(rt, cfg.get("table"), cfg.get("cond"), + int(cfg.get("limit") or 25)) + counts["found"] += len(found) + # ⛔ THE `review` BRANCH IS DELETED (wave 27 item 12, owner ruling R3), and it had + # already stopped being reachable one wave earlier — which is the part worth reading. + # `_without_retired_board` strips every `review` action out of a definition on the + # READ path, so `all_definitions` and this function have not seen one since the board + # was retired. What was left behind was a branch referencing `skey`, `stamp` and + # `ai_budget` — three names with NO DEFINITION anywhere in this module. It was not + # dead code that merely wasted space: it was a `NameError` held back by a migration + # rather than by a guard, and any change that let one stored `review` action through + # would have crashed the whole action walk for every record in that flow. + # ⚠ `ai_decide` and `review_audit` SURVIVE as library code with no caller — R3 keeps + # review "as an AI decision without lanes", and the cheap-first provider ladder behind + # it is real, working, measured work. They are PARKED, deliberately and in writing, + # not orphaned; see their own notes. + return False + + if actions: + for rid in list(row_ids or [])[:FLOOD_LIMIT]: + row = dict(rows.get(str(rid)) or {}) + if not row: + continue + # ⭐ D-191 — the web batch is built from THIS row's interpolated values, so it means + # nothing for the next one. Cleared here rather than inside `_walk`, which recurses + # into branches and would wipe a batch its own caller is still consuming. + web_done.clear() + _walk(actions, row, str(rid)) + # ⭐⭐ WAVE 30 · T08 — TWO FLUSHES, ONE `counts`, AND THE NOTES ARE CONCATENATED RATHER THAN + # OVERWRITTEN. `RUN_NOTES_KEY` is the one key both flushes legitimately produce, so a plain + # `counts.update(a); counts.update(b)` would drop every Instagram per-record reason the moment + # a flow also carried a TikTok step — silently, and precisely on the mixed flows where a + # person most needs to know which half failed. Every other key is prefixed and cannot collide. + _ig_out = _enrich_flush(rt, defn, username, enrich, log) + _tt_out = _tt_enrich_flush(rt, defn, username, tt, log) + _flush_notes = (list(_ig_out.pop(RUN_NOTES_KEY, None) or []) + + list(_tt_out.pop(RUN_NOTES_KEY, None) or [])) + counts.update(_ig_out) + counts.update(_tt_out) + if _flush_notes: + counts[RUN_NOTES_KEY] = _flush_notes + # ⭐ C5: the REALIZED numbers overwrite the walk's attempt count. `counts["created"]` was + # incremented once per create the flow decided to make; what landed is what the store says, + # and with `uniqueOn` on they are routinely different (a re-run of a scheduled flow matches + # every row it made last time — which is the whole point of the feature). + counts.update(_commit_action_writes(rt, table, patches, creates, username, log)) + return counts + + +def migrate_field_instagram(rt, defn): + """⭐ WAVE 25 · R4 — one stored `field_instagram` definition → a `plain` one carrying the + `enrich_instagram` action. Returns `(definition, changed)`. + + R4: "the ENRICH ACTION REPLACES the `field_instagram` KIND… migrate the one live + `field_instagram` automation to a plain flow carrying it; delete the kind and its label." + + ⛔ THE BINDING IS RESOLVED HERE, NOT LEFT TO THE FLAG, and this is the line the migration + turns on. `run_field_instagram` reads `cfg.urlField` **or falls back to `_auto_url_field`** — + so a live automation whose `urlField` is blank has been working off that fallback for months. + The enrich action deliberately has no such fallback (see `profile_field_key`), so migrating a + blank `urlField` verbatim would produce an automation that USED to work and now refuses. The + fallback is therefore evaluated ONCE, here, and the answer is written down as an explicit + binding — which is also the honest outcome: the column stops being implicit. + + ⚠ ONE BEHAVIOUR DOES CHANGE, AND IT IS THE POINT OF THE RULING RATHER THAN A REGRESSION. The + old kind wrote a STATUS STRING ("ok · 2026-08-06 · 12,400 followers") into `config.fieldKey`'s + column; the action writes the C1 PRESET CELLS instead. The status column is left in place and + simply stops being written — deleting somebody's column as part of a migration would be data + loss, and a stale cell beside a fresh `enriched_at` is readable for what it is. + + ⚠ PURE OVER THE DEFINITION apart from the one table READ. It writes nothing, so a caller can + run it to INSPECT what a migration would do — which is exactly how R4's "must be PROVEN + against the real stored definition" is meant to be satisfied. + """ + if (defn or {}).get("kind") != "field_instagram": + return defn, False + cfg = dict(defn.get("config") or {}) + target = str(cfg.get("targetTable") or "") + bound = str(cfg.get("urlField") or "").strip() + if not bound and target: + bound = str(_auto_url_field(ut_get(rt, target) or {}, cfg.get("fieldKey")) or "") + try: + max_posts = int(cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL) + except (TypeError, ValueError): + max_posts = 24 + act = {"id": "act_enrich", "kind": "enrich_instagram", "enabled": True, "when": None, + "config": {"profileField": bound, + "postMetrics": bool(cfg.get("postMetrics")), + "commentMetrics": bool(cfg.get("commentMetrics")), + "dryRun": bool(cfg.get("dryRun")), + "maxPosts": max(1, min(max_posts, 200))}} + flow = dict(defn.get("flow") or {}) + out = dict(defn) + out["kind"] = DEFAULT_KIND + # The enrich step goes FIRST — it is what the automation was, and every action the owner + # added afterwards was written expecting the capture to have happened. + out["flow"] = {"actions": [act] + list(flow.get("actions") or [])} + out["config"] = {"targetTable": target, + "targetLabel": str(cfg.get("targetLabel") or "")} + return out, True + + +def _enrich_flush(rt, defn, username, acc, log): + """C4: the enrich action's canonical IG tables, written ONCE for the whole run. + + Returns the counts the run reports. Empty when no enrich action ran, so a flow without one + pays nothing and its run history is unchanged (`run_now` merges only non-zero keys). + + ⛔ THE SAME WRITE PATH AS `run_field_instagram`, NOT A PARALLEL ONE: `ut_ensure` the three + tables, `upsert_rows` on their own keys with their own caps, one coalesced `rt.update`, then + the write-through to the platform master. Forking any of those would give the enrich action a + history that the metric fields could not see — and it is the LAST step, the write-through to + the platform master, that they read (W29-T34: `compute_metric_cells` → `ig_master.series_for`, + never this tenant's `ut_ig_snapshots`), so dropping that one line is the version of this fork + that would look harmless. + """ + if "unbound" in (acc.get("notes") or []): + # ⛔ DEBT D-79, SECOND HALF — THE SILENT FAILURE, AND IT WAS THE WORSE HALF. An enrich step + # on a database with no profile column `continue`s BEFORE `enrich["profiles"] += 1`, so + # this function used to return `{}`, `run_now` merged only non-zero keys, and the run + # committed **`ok` — "N records walked"** having written nothing at all. The sentence + # naming the fix went to `log()`, i.e. the server console, which no customer reads. That + # is "green over nothing" on the one path where somebody is waiting for data. + # ⚠ IT IS A COUNT, not a flag, so `run_now`'s existing non-zero merge carries it without a + # special case — and so the run entry itself records that this happened. + return {"enrichUnbound": 1} + # ⭐⭐ 2026-08-09 — THE NOTES SURVIVE A RUN THAT READ NOTHING, and that is not a detail. + # `if not acc["profiles"]: return {}` is exactly the branch a run takes when EVERY candidate + # was skipped as a known-dead handle — so the sentence explaining why the automation appears + # to do nothing would have been dropped on precisely the runs that most need it, and the + # owner would be back at "0 records walked, wtf". The selection note is produced before any + # profile is read and must outlive that early return. + if not acc.get("profiles"): + notes = list(acc.get("notes") or []) + return {RUN_NOTES_KEY: notes} if notes else {} + out = {"enriched": acc["ok"], "enrichBlocked": acc["blocked"]} + if acc.get("notes"): + out[RUN_NOTES_KEY] = list(acc["notes"]) + # ⭐ D-103's own prescription: "the block note survives on the RUN … not a status column, so + # no table gains a column it did not ask for". `RUN_NOTES_KEY` is that channel. + if not acc.get("dry"): + # The vendor's not-found verdicts, merged into engine state (never a tenant column and + # never a status string — W25/R4 retired those). Merged rather than replaced: a run that + # walked one record must not forget what earlier runs learned about the others. + if acc.get("gone"): + merged = dict(((defn or {}).get("state") or {}).get("enrichNotFound") or {}) + merged.update(acc["gone"]) + set_state(rt, str(defn.get("id") or ""), {"enrichNotFound": merged}) + pending_profiles = queue_pending_profile_snapshots( + rt, str(defn.get("id") or ""), acc.get("pendingProfiles") or []) + if pending_profiles: + out["enrichProfileBatchesPending"] = pending_profiles + queued = queue_pending_metric_snapshots(rt, str(defn.get("id") or ""), + acc.get("pending") or []) + if queued: + out["enrichMetricBatchesPending"] = queued + if acc.get("dry") or not (acc["snaps"] or acc["posts"] or acc["psnaps"] or acc["comments"]): + # A dry run resolves nothing and writes nothing — not even `ut_ensure`, which CREATES. + if acc.get("dry"): + out["enrichDryRun"] = acc["profiles"] + return out + tag = str(defn.get("id") or "") + profile_table = str(_flow_table(defn) or "") + graph = ensure_ig_graph(rt, username, tag, profile_table=profile_table) + snap_key, post_key, ps_key, comment_key = (graph[IG_SNAPSHOTS_TABLE], graph[IG_POSTS_TABLE], + graph[IG_POST_SNAPSHOTS_TABLE], graph[IG_COMMENTS_TABLE]) + missing = ut_missing(rt, snap_key, post_key, ps_key, comment_key) + snaps, c_snap = upsert_rows(dict((ut_get(rt, snap_key) or {}).get("rows") or {}), + acc["snaps"], "snapshot_key", cap=row_cap(snap_key)) + old_posts, collapsed_posts = dedupe_canonical_rows( + dict((ut_get(rt, post_key) or {}).get("rows") or {}), "shortcode", newest_by="measured_at") + posts, c_post = upsert_rows(old_posts, acc["posts"], "shortcode", cap=row_cap(post_key)) + c_post["duplicates"] += collapsed_posts + psnaps, c_ps = upsert_rows(dict((ut_get(rt, ps_key) or {}).get("rows") or {}), + acc["psnaps"], "post_snapshot_key", cap=row_cap(ps_key)) + old_comments, collapsed_comments = dedupe_canonical_rows( + dict((ut_get(rt, comment_key) or {}).get("rows") or {}), "comment_key") + comments, c_comments = upsert_rows(old_comments, acc["comments"], "comment_key", + cap=row_cap(comment_key)) + c_comments["duplicates"] += collapsed_comments + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + for k, rws in ((snap_key, snaps), (post_key, posts), (ps_key, psnaps), + (comment_key, comments)): + if cur.get(k) is not None: + cur[k]["rows"] = rws + _refresh_relations_inplace(cur, log=log) + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") # ONE coalesced update for all three tables + # LOUD, never silent (D-11): a full append table means the SERIES has stopped growing, which + # is the failure a chart cannot show you. + capped = c_snap["capped"] + c_post["capped"] + c_ps["capped"] + c_comments["capped"] + if capped: + out["enrichCapped"] = capped + log(f"[aios-auto] enrich_instagram: {capped} row(s) refused by a table's row cap") + if missing: + out["enrichMissingTables"] = len(missing) + log(f"[aios-auto] enrich_instagram: could not create {', '.join(missing)}") + out["enrichPoints"] = len(acc["snaps"]) + out["enrichPosts"] = c_post["inserted"] + out["enrichComments"] = c_comments["inserted"] + # C6/R2's write-through to the PLATFORM MASTER. Three postures, never conflated — the same + # contract `run_field_instagram` follows, because a pooled history with silent holes is worse + # than none. + try: + import ig_master + m_status, m_note = ig_master.append_run(getattr(rt, "key", ""), acc["snaps"], + acc["posts"], acc["psnaps"]) + if m_status == "error": + out["enrichMasterFailed"] = 1 + log(f"[aios-auto] enrich_instagram: the platform master copy FAILED. {m_note}; " + f"the tenant copy is complete and the next run re-appends") + elif m_status == "ok": + out["enrichMaster"] = len(acc["snaps"]) + len(acc["psnaps"]) + except Exception as e: # noqa: BLE001 + out["enrichMasterFailed"] = 1 + log(f"[aios-auto] enrich_instagram: master write-through raised " + f"{type(e).__name__}: {e}") + return out + + +#: How many candidate columns an unbound-enrich sentence names before it stops. Three, because the +#: sentence is read in a run log line: naming forty columns is the same as naming none. +UNBOUND_HINT_MAX = 3 + + +def _unbound_hint(rt, table_key): + """⭐ WAVE 30 · T14 — the *"…which column to mark"* half of D-79's sentence. + + Returns `" — this database's text columns are Handle, Who"` or `""`. Never a binding. + + ⛔ IT SUGGESTS AND DOES NOT RESOLVE, and the distinction is the whole reason this is a string + rather than a fallback. `profile_field_key`'s own docstring refuses to guess — *"the failure + would be a run that reports success having enriched from the wrong column … a wrong number is + harder to notice than a missing one"*. That argument is about BINDING. It says nothing against + telling a person, in the sentence they are already reading, which columns are even eligible: + the enrich still refuses, nothing is written, and the human makes the choice. + ⚠ `text` ONLY, matching the writer F shipped for the flag (`ColumnMenu`'s toggle is gated on + `editType === "text"`), so the sentence cannot offer a column the editor would then refuse. + """ + fields = (ut_get(rt, table_key) or {}).get("fields") or [] + # ⚠ AND A PRESET/LOCKED COLUMN IS NOT OFFERED. `Platform` is a `text` column on every preset + # profile table and carries `automation.preset` + an `editRole`, so the field editor refuses to + # retype it — offering it would send a person to a control that says no, which is exactly the + # claim the docstring above makes and did not honour on its first draft. + names = [str(f.get("label") or f.get("key") or "").strip() for f in fields + if str(f.get("type") or "text") == "text" + and not (f.get("automation") or {}).get("preset") + and not str(f.get("editRole") or "").strip() + and str(f.get("label") or f.get("key") or "").strip()] + if not names: + return "" + shown = ", ".join(names[:UNBOUND_HINT_MAX]) + more = len(names) - UNBOUND_HINT_MAX + return (f". This database's text columns are {shown}" + + (f" (+{more} more)" if more > 0 else "")) + + +def _tt_write_tables(rt, tag, username, snaps, posts, psnaps, comments, log=print): + """The four `ut_tt_*` tables, created-if-needed and written in ONE coalesced store update. + Returns `(inserted_by_table, capped, missing)`. + + ⭐⭐ WAVE 30 · D-156 — ONE WRITER, TWO CALLERS, AND THE SECOND CALLER IS WHY IT EXISTS. + This was the tail of `_tt_enrich_flush`, i.e. reachable only from an INLINE enrich. The + deferred collector needs exactly the same write, and the repo has already paid once for the + version where it did not: `top_up_views` lived inside `pull_profile_bd`, so it ran only when + the Posts scrape answered in time, and every DEFERRED Instagram run wrote posts with a blank + Views column. The fix there was this same shape — one function, two callers, so the inline and + deferred paths cannot answer differently — and copying the block instead would reintroduce the + class rather than the bug. + + ⭐ ONE LOOP OVER `(table, rows, key)` RATHER THAN FOUR HAND-WRITTEN BLOCKS, and the field list + comes from `TT_TABLE_FIELDS` — so a fifth `ut_tt_*` table is one tuple, and no table can be + created with a field list that disagrees with its own declaration. + ⚠ NO ROWS ⇒ NO DATABASE. Spawning `ut_tt_comments` on a collect that carried none gives a + person a database to watch never fill, which is the visible half of green-over-nothing. + """ + plan = [(TT_SNAPSHOTS_TABLE, snaps or [], "snapshot_key"), + (TT_POSTS_TABLE, posts or [], "shortcode"), + (TT_POST_SNAPSHOTS_TABLE, psnaps or [], "post_snapshot_key"), + (TT_COMMENTS_TABLE, comments or [], "comment_key")] + written, capped, missing = {}, 0, [] + inserted = {} + for table_key, rows_in, key_field in plan: + if not rows_in: + continue + # ⭐ R9 (W31-T32) — THE RUN PATH IS THE SITE THAT ACTUALLY CREATED TODAY'S TABLES, and it + # is the one the ticket's `how:` does not name. `waves/wave30/proof/tiktok-e2e-ut_tt_posts + # .png` shows `ut_tt_posts` with 8 real rows offering "+ New record" — those rows arrived + # HERE, not through the save path (which, until W31-T34, returned before the child spawn for + # a discovery automation). Stamping only the save site would have left every table that + # already exists unlocked. ⭐ And an EXISTING table does come forward on the next call: + # `ut_ensure`'s skip test carries `not (record_mode and have.get("recordMode") != + # record_mode)`, so no separate migration is needed. + real_key = ut_ensure(rt, TT_TABLE_LABELS[table_key], TT_TABLE_FIELDS[table_key], username, + key=table_key, flow_tag=tag, lock_fields=True, + record_mode=tt_record_mode(table_key)) + missing.extend(ut_missing(rt, real_key)) + merged, counts_ = upsert_rows(dict((ut_get(rt, real_key) or {}).get("rows") or {}), + rows_in, key_field, cap=row_cap(real_key)) + written[real_key] = merged + inserted[table_key] = counts_["inserted"] + capped += counts_["capped"] + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + for k, rws in written.items(): + if cur.get(k) is not None: + cur[k]["rows"] = rws + _refresh_relations_inplace(cur, log=log) + return cur + + if written: + rt.update(UT_STORE_KEY, _up, flush="sync") # ONE coalesced update for every table + return inserted, capped, missing + + +def _tt_enrich_flush(rt, defn, username, acc, log): + """⭐⭐ WAVE 30 · T08 — the TikTok enrich action's append table, written ONCE for the whole run. + + The twin of `_enrich_flush` and deliberately NOT a call into it. That function is bound to + Instagram at four points — `ensure_ig_graph`, the four `IG_*_TABLE` keys, `PRESET_*` and the + write-through to the platform Instagram master — and the last of those is the one that would + look harmless: `ig_master.append_run` is what the metric FIELDS read (W29-T34), so a TikTok + creator's followers appended there would become an Instagram number in a pooled history that + no later run could tell apart. + + ⛔ EVERY KEY IT RETURNS IS PREFIXED `tt…`. `apply_actions` merges both flushes into one + `counts`, so `enriched`/`enrichBlocked` would be one name for two networks' numbers and a + mixed flow would report whichever flushed last. + + ⭐ WAVE 30 · T10 — FOUR TABLES NOW, AND EACH IS CREATED ONLY WHEN IT HAS A ROW. Spawning + `ut_tt_posts` on a run that captured no posts gives a person a database to watch never fill, + which is the visible half of green-over-nothing and the same complaint that produced + `discover_default_table` (*"a SECOND, empty database"*). So the create rides the rows. + """ + if "unbound" in (acc.get("notes") or []): + # D-79 on the second network: a COUNT, so `run_now`'s existing non-zero merge carries it + # and the run entry itself records that the step could not run. Its own key, so the + # summary can name TikTok rather than borrowing Instagram's sentence. + return {"ttEnrichUnbound": 1} + if not acc.get("profiles"): + # The selection's note outlives a run that read nothing — the same reason as the IG side: + # "every candidate was skipped" is exactly the run whose silence needs explaining. + notes = list(acc.get("notes") or []) + return {RUN_NOTES_KEY: notes} if notes else {} + out = {"ttEnriched": acc["ok"], "ttEnrichBlocked": acc["blocked"]} + if acc.get("notes"): + out[RUN_NOTES_KEY] = list(acc["notes"]) + # ⭐⭐ W30 · D-156 — QUEUED BEFORE THE DRY-RUN AND EMPTY-ROWS RETURNS BELOW, and the order is + # the point. A run whose media batch DEFERRED has no posts and no comments to write, so it + # takes the `not (snaps or posts or ...)` exit — the exact run that owns a paid snapshot id. + # Filing after that return would have collected nothing, forever, which is the shape the + # inline-vs-deferred split keeps producing. + # ⚠ Its own count key (`tt…`), like every other key here: `apply_actions` merges both + # networks' flushes into ONE dict, so sharing Instagram's name would let a mixed flow report + # one network's pending batches under the other's. + if not acc.get("dry"): + queued = queue_pending_metric_snapshots(rt, str(defn.get("id") or ""), + acc.get("pending") or []) + if queued: + out["ttEnrichMetricBatchesPending"] = queued + if acc.get("dry") or not (acc.get("snaps") or acc.get("posts") or acc.get("psnaps") + or acc.get("comments")): + # A dry run resolves nothing and writes nothing — not even `ut_ensure`, which CREATES. + if acc.get("dry"): + out["ttEnrichDryRun"] = acc["profiles"] + return out + tag = str(defn.get("id") or "") + inserted, capped, missing = _tt_write_tables( + rt, tag, username, acc.get("snaps"), acc.get("posts"), acc.get("psnaps"), + acc.get("comments"), log) + # LOUD, never silent (D-11): a full append table means the SERIES has stopped growing, which + # is the failure a chart cannot show you. + if capped: + out["ttEnrichCapped"] = capped + log(f"[aios-auto] enrich_tiktok: {capped} row(s) refused by a table's row cap") + if missing: + out["ttEnrichMissingTables"] = len(missing) + log(f"[aios-auto] enrich_tiktok: could not create {', '.join(missing)}") + out["ttEnrichPoints"] = len(acc.get("snaps") or []) + if inserted.get(TT_POSTS_TABLE): + out["ttEnrichPosts"] = inserted[TT_POSTS_TABLE] + if inserted.get(TT_COMMENTS_TABLE): + out["ttEnrichComments"] = inserted[TT_COMMENTS_TABLE] + # ⛔ AND NO PLATFORM-MASTER WRITE-THROUGH, which is a deliberate absence rather than a missing + # line. `ig_master` is Instagram's pooled history and there is no TikTok equivalent yet; the + # tenant's own `ut_tt_snapshots` is the complete record today, and inventing a second store + # for a series nobody reads would be the fork this function exists to avoid. + return out + + +_PENDING_METRIC_DATASETS = frozenset((BD_DS_POSTS, BD_DS_REELS, BD_DS_COMMENTS)) +_PENDING_METRIC_KINDS = frozenset(("posts", "comments")) + +#: ⭐⭐ WAVE 30 · D-156 — THE TIKTOK HALF OF THE METRIC QUEUE, BUILT LAZILY FROM `connectors_tt`'s +#: OWN CONSTANTS. Re-declaring the two ids here would be a second copy of a vendor identifier that +#: nothing compares — the drift `MAX_UT_ROWS` demonstrated at 12x — so this reads them from the one +#: module that owns them, through `_tt_module()` because `connectors_tt` imports THIS module. +#: ⚠ It is a MAP rather than a set because on TikTok one dataset is exactly one kind, which is why +#: this network needs no `_tag_metric_deferrals` twin: the id the transport already recorded says +#: whether a batch is posts or comments, so nothing downstream has to be told twice. +_TT_METRIC_KIND_BY_DATASET = None + + +def tt_metric_kind(dataset_id): + """`"posts"` / `"comments"` for a TikTok media dataset id; `""` for anything else. + + ⛔ THE `""` IS LOAD-BEARING AND IS THE PLATFORM TEST. `collect_pending_metric_snapshots` + branches on it, so a dataset this map does not know keeps Instagram's mappers — which is the + safe direction, because Instagram's are what every stored pre-wave-30 task needs. + """ + global _TT_METRIC_KIND_BY_DATASET + if _TT_METRIC_KIND_BY_DATASET is None: + _tt = _tt_module() + _TT_METRIC_KIND_BY_DATASET = {str(_tt.TT_DS_POSTS): "posts", + str(_tt.TT_DS_COMMENTS): "comments"} + return _TT_METRIC_KIND_BY_DATASET.get(str(dataset_id or ""), "") + + +def _pending_metric_tasks(defn): + """Read validated, deduplicated paid metric snapshots from automation continuation state. + + A snapshot ID is a vendor-issued capability for work already paid for. It is intentionally + stored as engine state beside discovery's pending corpus snapshot, never in a Profile cell or + the user-editable flow. Invalid/stale shapes are ignored rather than sent back to a vendor + endpoint, and the collector never starts a second scrape request. + """ + raw = ((defn or {}).get("state") or {}).get("pendingMetricSnapshots") or [] + out, seen = [], set() + for item in raw if isinstance(raw, list) else []: + if not isinstance(item, dict): + continue + sid = str(item.get("snapshotId") or "").strip() + dataset = str(item.get("datasetId") or "").strip() + kind = str(item.get("kind") or "").strip() + handle = str(item.get("influencer") or "").strip().lstrip("@").lower() + # ⭐ W30 · D-156 — BOTH NETWORKS' MEDIA CORPORA ARE COLLECTABLE NOW. The membership test + # stays a WHITELIST (a snapshot id is a vendor capability that has already been paid for; + # accepting an unknown dataset would send our key at a corpus no mapper here can read), + # and TikTok's half is asked of the map that owns it rather than listed again. + if (not sid.startswith("sd_") + or (dataset not in _PENDING_METRIC_DATASETS and not tt_metric_kind(dataset)) + or kind not in _PENDING_METRIC_KINDS or not handle): + continue + key = (sid, dataset, kind, handle) + if key in seen: + continue + seen.add(key) + out.append({"snapshotId": sid, "datasetId": dataset, "kind": kind, + "influencer": handle, "requestedAt": str(item.get("requestedAt") or ""), + "lastChecked": str(item.get("lastChecked") or ""), + "lastNote": _s(item.get("lastNote"), 160)}) + return out + + +def queue_pending_metric_snapshots(rt, auto_id, pending): + """Durably retain new engagement snapshots without creating another paid provider request.""" + aid = str(auto_id or "").strip() + if not aid: + return 0 + incoming = _pending_metric_tasks({"state": {"pendingMetricSnapshots": pending}}) + if not incoming: + return 0 + added = [0] + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + definition = cur.get(aid) + if not isinstance(definition, dict): + return cur + state = definition.setdefault("state", {}) + existing = _pending_metric_tasks(definition) + known = {(x["snapshotId"], x["datasetId"], x["kind"], x["influencer"]) + for x in existing} + for task in incoming: + key = (task["snapshotId"], task["datasetId"], task["kind"], task["influencer"]) + if key not in known: + existing.append(task) + known.add(key) + added[0] += 1 + state["pendingMetricSnapshots"] = existing + return cur + + _store_update(rt, _up, flush="sync") + return added[0] + + +def _match_profile_row(rows, handle): + """THIS handle's row out of a snapshot that may now hold several. `None` when it is not there. + + ⭐⭐ 2026-08-09 — `rows[0]` WAS SAFE ONLY WHILE EVERY SNAPSHOT HELD EXACTLY ONE PROFILE. With + `_prime_enrich_batch` buying five URLs per call, one snapshot id serves up to five records, and + taking the first row would write ONE profile's followers onto all of them — a wrong number that + looks exactly like a right one. Identity comes off the row (`account`/`username`), which is the + same field `_bd_profile` reads first. + + ⚠ THE SINGLE-ROW FALLBACK IS DELIBERATE AND NARROW. Every task queued before this change points + at a one-URL snapshot, and some of those rows have an unreadable identity; for exactly that + shape the task's own handle is still the best evidence. It applies only when the snapshot holds + ONE row, so it can never mis-assign inside a batch. + """ + want = str(handle or "").strip().lstrip("@").lower() + usable = [n for n in (rows or []) if isinstance(n, dict)] + for node in usable: + got = str(_first(node, "account", "username", default="") or "").strip() + if got.lstrip("@").lower() == want and want: + return node + if len(usable) == 1: + return usable[0] + return None + + +def _pending_profile_tasks(defn): + """Validated, deduplicated deferred PROFILE snapshots from automation state. + + ⭐⭐ ITS OWN LIST, NOT `pendingMetricSnapshots`, and the separation is load-bearing rather + than tidy. `_pending_metric_tasks` filters on `datasetId in {posts, reels, comments}` and + `kind in {posts, comments}` — so a profile entry appended to that list is silently dropped to + zero by its own validator, and even if it survived, the collector would hand it to + `_write_collected_metric_rows`, a posts/comments writer that has nothing to do with a profile + row. Reusing the name would have shipped a green no-op of exactly the class this change + exists to remove. + + ⚠ A PROFILE TASK CARRIES ITS DESTINATION (`table` + `rowId`). A collected profile is written + back as PRESET CELLS onto the record that asked for it, so unlike a metric batch it cannot be + resolved from the handle alone: two databases may both hold `@x`. + """ + raw = ((defn or {}).get("state") or {}).get("pendingProfileSnapshots") or [] + out, seen = [], set() + for item in raw if isinstance(raw, list) else []: + if not isinstance(item, dict): + continue + sid = str(item.get("snapshotId") or "").strip() + dataset = str(item.get("datasetId") or "").strip() + handle = str(item.get("influencer") or "").strip().lstrip("@").lower() + table = str(item.get("table") or "").strip() + row_id = str(item.get("rowId") or "").strip() + # ⛔ `sd_` ONLY. A `snap_…` corpus id sent to `/datasets/v3/…` is a flat 404 about a + # snapshot that is alive (§2c), and a malformed id must never be handed back to a vendor + # endpoint at all. + if (not sid.startswith("sd_") or dataset != BD_DS_PROFILES or not handle + or not table or not row_id): + continue + if sid in seen: + continue + seen.add(sid) + out.append({"snapshotId": sid, "datasetId": dataset, "kind": "profile", + "influencer": handle, "table": table, "rowId": row_id, + "requestedAt": str(item.get("requestedAt") or ""), + "lastChecked": str(item.get("lastChecked") or ""), + "lastNote": _s(item.get("lastNote"), 200)}) + return out + + +def queue_pending_profile_snapshots(rt, auto_id, pending): + """Durably retain deferred profile snapshots. Starts no new paid request. Returns how many + were newly added.""" + aid = str(auto_id or "").strip() + if not aid: + return 0 + incoming = _pending_profile_tasks({"state": {"pendingProfileSnapshots": pending}}) + if not incoming: + return 0 + added = [0] + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + definition = cur.get(aid) + if not isinstance(definition, dict): + return cur + state = definition.setdefault("state", {}) + existing = _pending_profile_tasks(definition) + known = {x["snapshotId"] for x in existing} + for task in incoming: + if task["snapshotId"] not in known: + existing.append(task) + known.add(task["snapshotId"]) + added[0] += 1 + state["pendingProfileSnapshots"] = existing + return cur + + _store_update(rt, _up, flush="sync") + return added[0] + + +def collect_pending_profile_snapshots(rt, defn, username="automation", log=print, step=_no_step): + """Finish deferred PROFILE reads the tenant has already paid for. Starts no new scrape. + + ⭐⭐ 2026-08-09 — THE HALF THAT DID NOT EXIST. `bd_scrape` has always accepted a `deferred` + list, and every post/reel/comment call passed one; the PROFILE call did not, so a profile the + vendor took longer than `BD_SCRAPE_WAIT` to collect was billed and its snapshot id thrown + away — on every run, forever. MEASURED on nurilab: `collection_duration` 320 s against a + 180 s budget, two abandoned `sd_…` snapshots in two runs. + + ⛔ IT CLOSES A TASK THAT FINISHED EMPTY. A pending entry that can never resolve is the same + forever-loop wearing a different mask, so `bd_snapshot_progress` deciding `done` with zero + records ends the task with the vendor's reason attached — and, when the vendor blames the + target rather than itself, records the not-found verdict so the handle stops being re-bought. + """ + tasks = _pending_profile_tasks(defn) + if not tasks: + return ("ok", "No pending profile reads.", {}, [], {"source": "idle"}) + step(f"Collecting {len(tasks)} deferred profile read{'' if len(tasks) == 1 else 's'}") + remaining, patches, affected, notes = [], {}, [], [] + acc = {"snaps": [], "posts": [], "psnaps": [], "comments": [], "pending": [], + "pendingProfiles": [], "gone": {}, "profiles": 0, "ok": 0, "blocked": 0, + "notes": [], "dry": False} + ready = waiting = closed = 0 + # ⭐ ONE SNAPSHOT, ONE STATUS CALL, ONE FETCH. A batched read gives several tasks the SAME + # `snapshotId`, and asking the vendor about it once per task would spend the round trips the + # batching just saved. Memoised for this collection pass only — a snapshot's state must be + # re-read on the NEXT run, which is a fresh call and a fresh dict. + _progress_memo, _rows_memo = {}, {} + + def _progress_of(sid): + if sid not in _progress_memo: + _progress_memo[sid] = bd_snapshot_progress(sid) + return _progress_memo[sid] + + def _store_profile(task, profile, via): + """Write ONE collected profile — snapshot row + preset cells on the record that asked. + + ONE implementation, TWO callers (the primary snapshot and the backup rung below), so the + two paths cannot drift into writing different things — the same rule `top_up_views` + follows for the inline/deferred split. + """ + res = {"state": "ok", "profile": profile, "posts": [], "comments": [], + "via": via, "note": ""} + pulled = _iso() + snap_row, idents, metric_rows, comment_rows = capture_rows(res, pulled) + acc["snaps"].append(snap_row) + acc["posts"].extend(idents) + acc["psnaps"].extend(metric_rows) + acc["comments"].extend(comment_rows) + _act_row_patch(patches, task["table"], task["rowId"], preset_cells(res, pulled)) + affected.append(task["rowId"]) + + def _backup_profile(handle): + """The second rung, asked ONLY once the first has finished and delivered nothing. + + ⭐ 2026-08-09 (owner: *"when it errors like this, just route to APIfy"*). THE GAP THIS + CLOSES: `pull_profile` has walked `ig_profile -> (primary, backup)` since 2026-08-08, but + a profile that DEFERRED never came back through `pull_profile` — it came back here, and + this collector had no second rung at all. So the one path where the primary is most + likely to have failed was the one path with no fallback. + + ⛔ NEVER A TOP-UP. It runs only in the finished-and-empty branch, so a profile the primary + answered is never re-bought from a second vendor — the mistake the capability split exists + to prevent. A refusal returns None and the caller reports blocked exactly as before. + """ + try: + import providers as _p + if not _p.PROVIDERS["apify"].can("ig_profile"): + return None + prof, note = apify_profile(str(handle)) + except Exception as exc: # noqa: BLE001 — a backup must not raise + log(f"[aios-auto] backup profile rung failed for @{handle}: " + f"{type(exc).__name__}: {exc}") + return None + if prof and prof.get("followers") is not None: + return prof + return None + + def _rows_of(sid): + if sid not in _rows_memo: + payload, err = bd_call(f"{BD_PATH_SNAPSHOT}/{sid}", {"format": "json"}) + got = _bd_rows(payload) if not err else [] + if got and (_bd_deferral(payload) or + (len(got) == 1 and str(got[0].get("status") or "") in + ("running", "building", "collecting"))): + got = [] + _rows_memo[sid] = got + return _rows_memo[sid] + + for task in tasks: + stale_h = _hours_since(task.get("requestedAt")) + state, records, empty_note = _progress_of(task["snapshotId"]) + if state in ("done", "failed") and not records: + acc["profiles"] += 1 + # ⭐ ASK THE BACKUP BEFORE GIVING UP. The primary has FINISHED and delivered nothing, + # so there is nothing left to wait for and no risk of buying the same record twice. + backup = _backup_profile(task["influencer"]) + if backup is not None: + ready += 1 + acc["ok"] += 1 + _store_profile(task, backup, "apify") + note = (f"@{task['influencer']}: the primary source returned nothing, so a backup " + f"source supplied the profile ({backup.get('followers')} followers)") + notes.append(note) + acc["notes"].append(note) + continue + closed += 1 + acc["blocked"] += 1 + note = f"@{task['influencer']}: {_s(empty_note, 240)}" + notes.append(note) + acc["notes"].append(note) + # ⭐ `failed` = the vendor finished, collected nothing, and blamed the TARGET. On a + # profile request that means the account could not be reached at all, so the verdict + # is remembered and the selection stops re-buying it (R9: permanently, until a + # human edits the handle cell — see `clear_gone`). + # ⚠ `done`-with-zero is NOT remembered: "we found no matches" is a statement about + # the query, and turning it into "this account does not exist" would silently retire + # live handles. + if state == "failed": + acc["gone"][_gone_key({}, task["influencer"])] = { + "at": _iso(), "handle": task["influencer"], "note": _s(empty_note, 300)} + continue + rows = _rows_of(task["snapshotId"]) if state != "running" else [] + if not rows: + # ⚠ BOUNDED. A snapshot the vendor never finishes must not be polled until the end of + # time; after `PENDING_PROFILE_MAX_HOURS` it is dropped WITH a sentence, never + # silently. An unbounded queue is the forever-loop this change removes, inverted. + if stale_h is not None and stale_h >= PENDING_PROFILE_MAX_HOURS: + closed += 1 + note = (f"@{task['influencer']}: the source never finished the profile read " + f"queued {int(stale_h)}h ago ({task['snapshotId']}). It was dropped; " + f"the next run will ask again") + notes.append(note) + acc["notes"].append(note) + continue + waiting += 1 + remaining.append({**task, "lastChecked": _iso(), + "lastNote": _s("still building", 200)}) + continue + # ⛔ THIS HANDLE'S ROW, NOT THE FIRST ONE — see `_match_profile_row`. A batched snapshot + # holds several profiles and `rows[0]` would write one creator's numbers onto every record + # in the chunk. + node = _match_profile_row(rows, task["influencer"]) + if node is None: + closed += 1 + acc["profiles"] += 1 + acc["blocked"] += 1 + note = (f"@{task['influencer']}: the source delivered {len(rows)} profile" + f"{'' if len(rows) == 1 else 's'} for that batch, none of them this handle. " + f"it was dropped from the batch and the next run will ask again") + notes.append(note) + acc["notes"].append(note) + continue + ready += 1 + acc["profiles"] += 1 + profile = _bd_profile(node, task["influencer"]) + if profile.get("followers") is None and profile.get("following") is None: + acc["blocked"] += 1 + note = (f"@{task['influencer']}: the source delivered the profile but no " + f"follower/following counts were readable in it") + notes.append(note) + acc["notes"].append(note) + continue + acc["ok"] += 1 + _store_profile(task, profile, "brightdata:deferred") + notes.append(f"@{task['influencer']}: collected the profile the source had already been " + f"paid for ({profile.get('followers')} followers)") + + set_state(rt, str(defn.get("id") or ""), + {"pendingProfileSnapshots": remaining or None}) + counts = _enrich_flush(rt, defn, username, acc, log) + counts.pop(RUN_NOTES_KEY, None) # this function owns the note list below + counts.update(_commit_action_writes(rt, str(_flow_table(defn) or ""), patches, {}, + username, log)) + counts.update({"profileBatchesCollected": ready, "profileBatchesPending": waiting, + "profileBatchesEmpty": closed}) + if notes: + counts[RUN_NOTES_KEY] = notes + head = (f"{ready} deferred profile read{'' if ready == 1 else 's'} collected" + if ready else "no deferred profile read was ready") + tail = "".join([f"; {closed} finished with nothing to collect" if closed else "", + f"; {waiting} still building" if waiting else ""]) + state = "ok" if ready and not closed else "partial" + return (state, head + tail, counts, affected, + {"source": "ok" if ready else "partial", "write": "ok" if ready else "idle"}) + + +def _write_collected_metric_rows(rt, defn, username, idents, snapshots, comments, log): + """Write a completed metric snapshot through the canonical Post/Comment graph once.""" + if not (idents or snapshots or comments): + return {"posts": 0, "snapshots": 0, "comments": 0} + profile_table = str(_flow_table(defn) or "") + if not profile_table: + raise Refused("the pending engagement snapshot has no Profile database to link to") + graph = ensure_ig_graph(rt, username, str(defn.get("id") or ""), + profile_table=profile_table) + post_key, ps_key, comment_key = (graph[IG_POSTS_TABLE], graph[IG_POST_SNAPSHOTS_TABLE], + graph[IG_COMMENTS_TABLE]) + old_posts, collapsed_posts = dedupe_canonical_rows( + dict((ut_get(rt, post_key) or {}).get("rows") or {}), "shortcode", newest_by="measured_at") + posts, c_post = upsert_rows(old_posts, idents, "shortcode", cap=row_cap(post_key)) + c_post["duplicates"] += collapsed_posts + psnaps, c_ps = upsert_rows(dict((ut_get(rt, ps_key) or {}).get("rows") or {}), snapshots, + "post_snapshot_key", cap=row_cap(ps_key)) + old_comments, collapsed_comments = dedupe_canonical_rows( + dict((ut_get(rt, comment_key) or {}).get("rows") or {}), "comment_key") + comments_rows, c_comments = upsert_rows(old_comments, comments, "comment_key", + cap=row_cap(comment_key)) + c_comments["duplicates"] += collapsed_comments + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + for key, rows in ((post_key, posts), (ps_key, psnaps), (comment_key, comments_rows)): + if cur.get(key) is not None: + cur[key]["rows"] = rows + _refresh_relations_inplace(cur, log=log) + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") + try: + import ig_master + status, note = ig_master.append_run(getattr(rt, "key", ""), [], idents, snapshots) + if status == "error": + log(f"[aios-auto] deferred post metrics master copy FAILED: {note}") + except Exception as exc: # noqa: BLE001 + log(f"[aios-auto] deferred post metrics master copy FAILED: {type(exc).__name__}: {exc}") + return {"posts": c_post["inserted"], "snapshots": c_ps["inserted"], + "comments": c_comments["inserted"]} + + +def collect_pending_metric_snapshots(rt, defn, username="automation", log=print, step=_no_step): + """Collect deferred paid Post/Reel/Comment snapshots; never launches a new scrape request.""" + tasks = _pending_metric_tasks(defn) + if not tasks: + return ("ok", "No pending post-engagement snapshots.", {}, [], {"capture_posts": "idle"}) + step(f"Collecting {len(tasks)} post-engagement batch{'' if len(tasks) == 1 else 'es'}") + remaining, idents, snapshots, comments = [], [], [], [] + # ⭐ W30 · D-156 — TikTok's rows accumulate SEPARATELY and are written by TikTok's own writer. + # One queue can hold both networks' snapshots (they are keyed by dataset), but the two write + # paths address different tables and neither may touch the other's. + tt_collected = {"posts": [], "psnaps": [], "comments": []} + ready, waiting, closed, run_notes = 0, 0, 0, [] + for task in tasks: + # ⭐⭐ 2026-08-09 — ASK THE STATUS DOCUMENT, NOT THE ROWS. `building` used to be + # `not rows or …`, so a snapshot the vendor had FINISHED with zero records was re-queued + # as "still building" on every tick — forever, because nothing about it would ever + # change. That is the same forever-loop the profile path was measured in, one dataset + # over, and it was latent here the whole time. + state, records, empty_note = bd_snapshot_progress(task["snapshotId"]) + if state in ("done", "failed") and not records: + # ⛔ CLOSED, NOT RE-QUEUED. The vendor is finished and there is nothing to collect; + # keeping the entry would be a pending task that can never resolve. + closed += 1 + run_notes.append(f"{task['influencer']}: {_s(empty_note, 160)}") + continue + payload, note = bd_call(f"{BD_PATH_SNAPSHOT}/{task['snapshotId']}", {"format": "json"}) + rows = _bd_rows(payload) if not note else [] + building = (state == "running" or not rows or _bd_deferral(payload) or + (len(rows) == 1 and str(rows[0].get("status") or "") in + ("running", "building", "collecting"))) + if building: + remaining.append({**task, "lastChecked": _iso(), + "lastNote": _s(note or "still building", 160)}) + waiting += 1 + continue + ready += 1 + pulled = _iso() + # ⭐⭐ WAVE 30 · D-156 — THE TIKTOK ARM. Before this, `_PENDING_METRIC_DATASETS` was + # Instagram's three, so a TikTok media batch the vendor deferred was paid for and + # collected by NOTHING — the note carried the snapshot id and a human was the only + # collector. It routes through `connectors_tt`'s own mappers and `tt_capture_rows`, + # never Instagram's, because every one of those stamps `PLATFORM_INSTAGRAM`. + # ⛔ TWO THINGS THE INSTAGRAM ARM DOES THAT THIS ONE MUST NOT, both deliberate: + # * NO `top_up_views`. TikTok's `play_count` arrives inline on the posts row, so there + # is no views capability to route to; calling it would buy an Instagram permalink. + # * NO `ig_master.append_run`. There is no TikTok master by design (`_tt_enrich_flush`), + # and a write-through would put a TikTok creator into Instagram's pooled history, + # which the metric FIELDS read — a wrong number in a permanent series. + if tt_metric_kind(task["datasetId"]): + _tt = _tt_module() + who = task["influencer"] + if task["kind"] == "posts": + mapped = [m for m in (_tt.normalize_post(r) for r in rows) if m] + for m in mapped: + # The snapshot was bought from THIS profile's own permalinks, so the backlink + # is a fact of the call even when a vendor row omits the author field. + m.setdefault("influencer_key", who) + res_tt = {"profile": {"handle": who}, "posts": mapped, "comments": []} + else: + mapped = [m for m in (_tt.normalize_comment(r) for r in rows) if m] + res_tt = {"profile": {"handle": who}, "posts": [], "comments": mapped} + tt_idents, tt_metrics, tt_comments = tt_capture_rows(res_tt, pulled) + tt_collected["posts"].extend(tt_idents) + tt_collected["psnaps"].extend(tt_metrics) + tt_collected["comments"].extend(tt_comments) + continue + if task["kind"] == "posts": + mapped_posts = [] + for raw in rows: + post = _bd_post_metrics(raw) + if post: + # The snapshot was created from this Profile's canonical post URLs. Keep + # that explicit backlink even when a vendor response omits `user_posted`. + post["influencer_key"] = task["influencer"] + mapped_posts.append(post) + if mapped_posts: + # ⭐⭐ 2026-08-09 — THE VIEWS TOP-UP RUNS HERE TOO, and its absence is why the + # owner's `theresalearns` run filled every column except Views. + # + # ⛔ Bright Data is DECLARED INCAPABLE of `ig_post_views` (`providers.py`), so a + # Posts row physically cannot carry a view count — MEASURED on the stored + # payloads: 12 rows, `content_type: "Reel"`, and no view/play key in any of them. + # Views only ever comes from the Apify capability. That top-up lived INSIDE + # `pull_profile_bd`, so it ran only when the Posts scrape answered within the wait + # budget; when the batch deferred — which is routine, and what happened here — the + # rows came back through THIS function and Apify was never asked. + # ⇒ `top_up_views` is now one function with two callers rather than a copy, so + # the inline and deferred paths cannot answer this differently again. + # ⚠ It mutates `mapped_posts` in place and must run BEFORE `capture_rows`, which + # is what freezes the values into the post + snapshot rows. + v_note = top_up_views(mapped_posts, log=log) + if v_note: + run_notes.append(f"@{task['influencer']}: {v_note}") + _unused, post_rows, metric_rows, embedded = capture_rows( + {"state": "ok", "profile": {"username": task["influencer"]}, + "posts": mapped_posts, "comments": [], "via": "brightdata:deferred"}, pulled) + idents.extend(post_rows) + snapshots.extend(metric_rows) + comments.extend(embedded) + else: + for raw in rows: + comment = _bd_comment(raw, influencer_key=task["influencer"]) + if comment: + comments.append(comment) + + set_state(rt, str(defn.get("id") or ""), + {"pendingMetricSnapshots": remaining or None}) + written = _write_collected_metric_rows(rt, defn, username, idents, snapshots, comments, log) + counts = {"metricBatchesCollected": ready, "metricBatchesPending": waiting, + "metricBatchesEmpty": closed, + "postEngagementSnapshots": written["snapshots"], + "commentsCollected": written["comments"]} + # ⭐ W30 · D-156 — the TikTok write, through the SAME function the inline enrich uses, so a + # collected batch and an inline one cannot land differently. Its counts carry the `tt` prefix + # for the reason every other TikTok count does. + if any(tt_collected.values()): + tt_inserted, tt_capped, tt_missing = _tt_write_tables( + rt, str(defn.get("id") or ""), username, [], tt_collected["posts"], + tt_collected["psnaps"], tt_collected["comments"], log) + for key, name in ((TT_POSTS_TABLE, "ttPostsCollected"), + (TT_POST_SNAPSHOTS_TABLE, "ttPostSnapshotsCollected"), + (TT_COMMENTS_TABLE, "ttCommentsCollected")): + if tt_inserted.get(key): + counts[name] = tt_inserted[key] + if tt_capped: + counts["ttCollectCapped"] = tt_capped + log(f"[aios-auto] collect: {tt_capped} TikTok row(s) refused by a table's row cap") + if tt_missing: + log(f"[aios-auto] collect: could not create {', '.join(tt_missing)}") + # ⚠ THE RUNNER CONTRACT STAYS A 5-TUPLE and the notes ride in `counts` under the reserved + # `RUN_NOTES_KEY`, which `run_now` pops. Widening the tuple for one runner would make four + # other call sites disagree about the shape of a run — and `_commit_run` already drops + # non-numeric count values, so a pop that is ever missed degrades to today's behaviour rather + # than to a crash. + if run_notes: + counts[RUN_NOTES_KEY] = run_notes + # ⚠ THE EMPTY ONES ARE NAMED, not silently dropped. A batch that finished with no records is + # a real outcome the tenant paid for and it must read as an answer, not as a disappearance. + tail = (f"; {closed} finished with nothing to collect" if closed else "") + if waiting: + return ("partial", + f"{ready} post-engagement batch{'' if ready == 1 else 'es'} collected; " + f"{waiting} still building and will be collected automatically{tail}", + counts, [], {"capture_posts": "partial", "write": "ok"}) + return ("ok", f"{ready} post-engagement batch{'' if ready == 1 else 'es'} collected{tail}", + counts, [], {"capture_posts": "ok", "write": "ok"}) + + +def ai_decide(rt, defn, act, row, row_id="", log=print): + """R4/C6: let the model pick this card's next stage. Returns the chosen stage LABEL, or "" + to leave the card for a person. + + ⛔ FAIL-CLOSED IN EVERY DIRECTION: no provider configured, a network failure, a malformed + answer, or a label the review does not offer all return "" — and "" means the card sits at + the review gate exactly as it would with no AI at all. The feature can be broken, absent or + wrong and the worst outcome is a human doing the work. + + ⚠⚠ PARKED SINCE WAVE 27 — THIS FUNCTION HAS NO CALLER, AND THAT IS RECORDED RATHER THAN + ACCIDENTAL. Its one caller was the `review` branch of the action walk, deleted with the board + under R3, which keeps review "as an AI decision without lanes". What is parked is genuinely + worth parking: the cheap-first provider ladder in `ai_review` (groq → cerebras → openrouter → + anthropic), the fail-closed posture above, and the audit shape `review_audit` writes. What is + MISSING is only the door — an action kind that asks a question and takes an answer, without a + stage column to write it into. **Do not delete this in a dead-code sweep without reading that + sentence first**; equally, do not treat it as shipped — nothing reaches it today. + """ + cfg = act.get("config") or {} + options = list(cfg.get("next") or []) + if not options: + return "" + try: + import ai_review + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] ai review unavailable: {type(e).__name__}: {e}") + return "" + fields = [f.get("key") for f in + (ut_get(rt, ((defn.get("config") or {}).get("targetTable") + or (defn.get("trigger") or {}).get("table") or "")) or {} + ).get("fields") or []] + # ⭐ W35 · CONTRACT C7 (`NOTE E-16`) — ATTRIBUTE THE SPEND. `st=` is what lets the ledger write + # to the right tenant's store and `user=` is who it is billed to; without them the call is + # counted as UNATTRIBUTED, which is a meter that reports a total nobody can act on. + # ⚠ `createdBy` is the honest actor here: an AI review decision is made ON BEHALF of the + # automation, by a scheduler, with no person at the keyboard. Naming whoever last edited it + # would attribute a nightly run to an editor who was asleep. + choice, meta = ai_review.decide(prompt=cfg.get("prompt") or "", options=options, + row=row, fields=[f for f in fields if f], + label=cfg.get("label") or "Review", + st=rt, user=str(defn.get("createdBy") or "")) + if not choice: + if meta.get("problem"): + log(f"[aios-auto] ai review declined to answer: {meta['problem']}") + return "" + review_audit(rt, defn.get("id"), row_id, cfg.get("label") or "Review", + choice, meta.get("provider") or "ai", by="ai", + note=meta.get("reason") or "", model=meta.get("model") or "") + return choice + + +def find_records(rt, table_key, cond, limit=25): + """The `find_records` action's read: matching row ids, bounded and DISCLOSED (the caller puts + the count in the run log, where it drills — [[no-unverifiable-aggregates]]).""" + rows = (ut_get(rt, str(table_key or "")) or {}).get("rows") or {} + out = [] + for rid, row in rows.items(): + if lane_match(cond, row or {}): + out.append(str(rid)) + if len(out) >= max(1, min(int(limit or 25), FIND_LIMIT_MAX)): + break + return out + + +def _commit_action_writes(rt, table, patches, creates, username, log): + """The ONE write. Row patches merge into the target's rows; creates land in their own tables — + APPENDED, or UPSERTED on the action's `uniqueOn` key (C5 / owner ruling R1a). + + Returns the REALIZED create counts, and returning them is the point rather than a + convenience. `apply_actions` counts an ATTEMPT per create while it walks the records; only + this function knows how many of those became rows, how many matched one that was already + there, and how many the cap refused. A run that reported the attempt as "created" would be + the summary-disagrees-with-what-happened defect this module names in three other places. + + ⛔ THE UPSERT IS `upsert_rows`, NOT A MATCH LOOP WRITTEN HERE. D-6 closed on "ONE + implementation repo-wide" after a second one was deleted from `core/user_tables.py`; hand + rolling a third inside this function is that debt returning with a new name — and it would + quietly diverge on the two rules that took a wave each to get right (an orphan is COUNTED, + NEVER DELETED, and `capped` is its own count rather than folded into `skipped`). + """ + out = {"created": 0, "createUpdated": 0, "createUnchanged": 0, + "createCapped": 0, "createSkipped": 0} + if not patches and not creates: + return out + # ⭐ PATCHES ARE STAGED, NOT WRITTEN YET (2026-08-06). They used to commit here, one + # `ut_write_rows` per patched table, and that was free while the only patcher was a review + # gate on a table no create action touched. Owner item 1 stamps EVERY walked record with the + # step it reached, so the flow's own table is now patched on essentially every run — and a + # flow that also creates into that same table would have committed it TWICE per run, against + # the 20 s flush floor and the 256-commits/hr repo budget this module is shaped around. + # Staged into `staged` and handed to the creates pass, which writes each table exactly once. + staged = {} + for tkey, rowpatch in (patches or {}).items(): + cur = dict((ut_get(rt, tkey) or {}).get("rows") or {}) + for rid, vals in rowpatch.items(): + cur[str(rid)] = {**(cur.get(str(rid)) or {}), **vals} + staged[str(tkey)] = cur + # ⛔ ONE WRITE PER TABLE, even when several actions target it under different keys. The + # accumulator is keyed by (table, uniqueOn), so a naive loop would call `ut_write_rows` once + # per GROUP — two store commits for one table, against the 20 s flush floor and the 256/hr + # repo budget this whole module is shaped around. + by_table = {} + for (tkey, unique), new_rows in (creates or {}).items(): + by_table.setdefault(str(tkey), []).append((str(unique or ""), new_rows)) + for tkey, groups in by_table.items(): + t = ut_get(rt, tkey) + if t is None: + n = sum(len(r) for _u, r in groups) + log(f"[aios-auto] create_record: {tkey} no longer exists. {n} skipped") + out["createSkipped"] += n + continue + # The PATCHED rows when this table was also stamped this run, so the creates land on top + # of the stamp rather than on a copy of the store that predates it. + cur = staged.pop(tkey, None) + cur = dict(t.get("rows") or {}) if cur is None else cur + cap = row_cap(tkey) + for unique, new_rows in groups: + if unique: + cur, c = upsert_rows(cur, new_rows, unique, cap=cap) + out["created"] += c["inserted"] + out["createUpdated"] += c["updated"] + out["createUnchanged"] += c["unchanged"] + out["createCapped"] += c["capped"] + # ⚠ `skipped` here means "this row had no value for the unique key", which for a + # create action is a mapped value that interpolated to nothing — worth surfacing, + # because the symptom is otherwise a run that says it created less than it walked. + out["createSkipped"] += c["skipped"] + if c["capped"]: + log(f"[aios-auto] create_record: {tkey} at its {cap}-row cap. " + f"{c['capped']} row(s) not written") + continue + nxt = max([int(r) for r in cur if str(r).isdigit()] or [0]) + 1 + for i, vals in enumerate(new_rows): + if len(cur) >= cap: + # THE [:N] HONESTY RULE ([[no-unverifiable-aggregates]]): the number DROPPED is + # named, here and in the run's counts. This used to `break` with a log line + # that said the cap was hit and never said how much was lost. + out["createCapped"] += len(new_rows) - i + log(f"[aios-auto] create_record: {tkey} at its {cap}-row cap. " + f"{len(new_rows) - i} row(s) not written") + break + cur[str(nxt)] = dict(vals) + nxt += 1 + out["created"] += 1 + ut_write_rows(rt, tkey, cur) + # Whatever the creates pass did NOT claim: tables this run only STAMPED. Written last and + # once each, so "one store write per table" holds whether a table was patched, created into, + # or both. + for tkey, cur in staged.items(): + ut_write_rows(rt, tkey, cur) + return out + + +def _lane_sentence(cond, top=True): + """One condition tree → the sentence a step's `detail` carries on the canvas. + + C4: a GROUP renders as its children joined by "and"/"or" and parenthesised when nested, so a + label never claims a flat comparison the tree does not make. The SERVER composes it, for the + same reason it composes every other `detail` — a client paraphrase of a structure the engine + evaluates is a second implementation of the same sentence, free to drift from it. + + ⚠ The name is board-era ("lane") and the board is gone; the caller is `graph()`, which is + live. Renamed nothing on purpose: this string is compared in a gate and read in a log, and a + rename would be churn on a working function to fix a word. + """ + if cond is None: + return "Everything else" if top else "" + if isinstance(cond, dict): + for key, joiner in (("all", " and "), ("any", " or ")): + if key in cond: + parts = [_lane_sentence(c, False) for c in cond.get(key) or []] + parts = [p for p in parts if p] + if not parts: + return "" + inner = joiner.join(parts) + return inner if top or len(parts) == 1 else f"({inner})" + v = cond.get("value") + return f"{cond.get('field')} {cond.get('op')}" + ("" if v is None else f" {v}") + + +def _rid_num(rid): + return int(rid) if str(rid).isdigit() else 10 ** 9 + + +#: How many review decisions an automation remembers. Bounded like `runs` — an audit that can +#: grow a definition without limit is a serialisation cost wearing a compliance hat. +MAX_REVIEWS = 100 + + +def review_audit(rt, auto_id, row_id, from_label, to_label, username, + by="user", note="", model=""): + """C3-A2(5): a review decision is AUDITED — who moved which card where, when. Appended to + the definition (newest first, bounded). + + ⭐ WAVE 23 (R4/C6): an AI decision writes THE SAME ROW with `by: "ai"` plus the model that + made it and the one-line reason it gave. One audit log, not two — a reader asking "who + decided this card" must not have to know there are two places to look, and the moment an + AI decision is invisible beside a human one the log stops being an audit. + + ⚠ PARKED SINCE WAVE 27, with `ai_decide` and for the same reason. Both of its doors are gone: + `move_card` was deleted with the board (R3), and the grid door in `core.grid_events` wrote + this shape off a stage field's `flowId` — a field the migration now drops. Kept because the + SHAPE is the contract a future decision action would write, and re-deriving an audit format + is how two of them end up existing. + """ + entry = {"ts": _iso(), "user": _s(username, 80), "rowId": str(row_id), + "from": _s(from_label, 60), "to": _s(to_label, 60), + "by": "ai" if by == "ai" else "user"} + if note: + entry["note"] = _s(note, 300) + if model: + entry["model"] = _s(model, 60) + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + d = cur.get(str(auto_id)) + if d is not None: + d["reviews"] = ([entry] + list(d.get("reviews") or []))[:MAX_REVIEWS] + return cur + + _store_update(rt, _up, flush="sync") + + +# --------------------------------------------------------------------------------------------- +# METRIC FIELDS (wave 22, contract C7) + THE SUBJECT PURGE (D-24) +# --------------------------------------------------------------------------------------------- +# A metric field is `{measure, window, agg}` over the MASTER series (C6's pooling benefit — +# the value reflects every pull the platform has, not just this tenant's). Computed by the +# ENGINE and stored as machine cells (C7 amendment: run+tick time — the ut wire lives in files +# no session owns this wave), human-write-refused at the grid door, and every number drills to +# the exact snapshot rows behind it. +# +# ⛔ NO DATA IS BLANK, NEVER ZERO. A handle the master has never seen, a window with no +# snapshots in it, a post series with no measured engagement — all read as an EMPTY cell. Zero +# is a measurement ("they have none"); blank is an admission ("we have not looked / it was not +# readable") — the `_bd_posts_count` law, one layer up. + +METRIC_MEASURES = ("followers", "avg_engagement", "likes", "comments") +METRIC_WINDOWS = ("latest", "last_3_posts", "last_7d", "last_30d") +#: Which measures read the PROFILE series vs the POST series — and which windows/aggs each +#: side can answer. A profile count over `last_3_posts` is a question the data cannot answer; +#: refused at clean time, never bent (mirrored in `core.user_tables`, gate-pinned). +PROFILE_MEASURES = ("followers", "avg_engagement") +METRIC_AGGS = ("avg", "sum", "latest") + + +def metric_value(series, measure, window, agg="", today=None): + """One metric over one handle's master series → `(value_string_or_None, drill_rows)`. + + `today` is a PARAMETER ([[date-window-vocabulary]]) — the caller decides the reference + day; post-count windows count back from the newest post. `drill_rows` are the exact rows + the number came from, so the route can honour [[no-unverifiable-aggregates]] without + recomputing differently.""" + series = series or {} + today = today or _now() + if measure in PROFILE_MEASURES: + rows = [r for r in series.get("snapshots") or [] + if str(r.get(measure) if r.get(measure) is not None else "").strip() != ""] + if window in ("last_7d", "last_30d"): + days = 7 if window == "last_7d" else 30 + floor = today - _dt.timedelta(days=days) + rows = [r for r in rows + if (_parse_iso(r.get("pulled_at")) or _dt.datetime.min) >= floor] + if not rows: + return None, [] + if window == "latest" or agg == "latest": + picked = [rows[-1]] + else: + picked = rows + vals = [_lane_num(r.get(measure)) for r in picked] + vals = [v for v in vals if v is not None] + if not vals: + return None, [] + out = vals[-1] if (window == "latest" or agg == "latest") else \ + (sum(vals) if agg == "sum" else sum(vals) / len(vals)) + if measure == "avg_engagement": + # The vendor's rate is 0-1; the pct cell renders POINTS (the semantic-pct vs + # transform-pct scar) — scaled exactly once, here. + return f"{out * 100:.2f}", picked + return (f"{out:.0f}" if float(out).is_integer() else f"{out:.2f}"), picked + # --- post measures: select POSTS by window, then each post's LATEST measured snapshot. + posts = [p for p in series.get("posts") or [] if str(p.get("posted_at") or "").strip()] + posts.sort(key=lambda p: str(p.get("posted_at"))) + if window == "latest": + picked_posts = posts[-1:] + elif window == "last_3_posts": + picked_posts = posts[-3:] + else: + days = 7 if window == "last_7d" else 30 + floor = today - _dt.timedelta(days=days) + picked_posts = [p for p in posts + if (_parse_iso(str(p.get("posted_at")).replace(" ", "T")) + or _dt.datetime.min) >= floor] + vals, drill = [], [] + for p in picked_posts: + snaps = (series.get("postSnapshots") or {}).get(str(p.get("shortcode") or "")) or [] + for snap in reversed(snaps): + raw = snap.get(measure) + v = _lane_num(raw) + if v is not None and str(raw).strip() != "": + vals.append(v) + drill.append(snap) + break + if not vals: + return None, [] + out = sum(vals) if (agg or "sum") == "sum" else \ + (vals[-1] if agg == "latest" else sum(vals) / len(vals)) + return (f"{out:.0f}" if float(out).is_integer() else f"{out:.2f}"), drill + + +def metric_fields_of_table(t): + return [f for f in ((t or {}).get("fields") or []) if isinstance(f.get("metric"), dict)] + + +def _table_handle(row, url_field): + return str(row.get("handle") or ig_handle(str(row.get(url_field or "") or "")) + or "").strip().lower() + + +def compute_metric_cells(rt, table_key, today=None): + """Recompute every metric cell on ONE table from the master series. One coalesced write, + only when something actually changed (the flush-ceiling law); zero reads when the table + has no metric fields or the master is off. Returns the number of rows touched.""" + t = ut_get(rt, table_key) + mfields = metric_fields_of_table(t) + if not mfields: + return 0 + import ig_master + if not ig_master.configured(): + return 0 + url_field = next((f.get("key") for f in (t.get("fields") or []) + if f.get("type") == "url"), "") + rows = t.get("rows") or {} + handles = {rid: _table_handle(row or {}, url_field) for rid, row in rows.items()} + series = ig_master.series_for({h for h in handles.values() if h}) + changes = {} + for rid, row in rows.items(): + s = series.get(handles.get(rid) or "") + for f in mfields: + bag = f["metric"] + val, _drill = (metric_value(s, bag.get("measure"), bag.get("window"), + bag.get("agg") or "", today=today) + if s else (None, [])) + want = "" if val is None else str(val) + if str((row or {}).get(f["key"], "")) != want: + changes.setdefault(str(rid), {})[f["key"]] = want + if not changes: + return 0 + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + tt = cur.get(table_key) + if tt is not None: + for rid, vals in changes.items(): + tt.setdefault("rows", {}).setdefault(rid, {}).update(vals) + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") + return len(changes) + + +# ── ⭐⭐ THE RELATIONAL PASS (2026-08-07) — derived LINK cells and ROLLUP cells ──────────────── +# +# Owner: *"adding relational database function with links and rollups… exactly like how Airtable +# does it… and this rollup needs to have formula that we can use to calculate things like average +# Views over last N posts."* +# +# ⛔ WHY THIS IS SERVER-SIDE AND MATERIALISED, when `formula` is client-side and is not. A formula +# reads ONE ROW; a rollup reads ANOTHER TABLE'S ROWS, which the client has not loaded and must not +# have to. So this rides `compute_metric_cells`' pattern exactly — recompute, diff, ONE coalesced +# write only when something changed — and inherits its flush-ceiling discipline for free. +# +# ⭐ AND R1's "ONE STORE FOR ONE SERIES" SURVIVES, which is the thing to check before touching +# this. The authoritative post record is `ut_ig_posts`; the authoritative engagement series is +# `ut_ig_post_snapshots`. A rollup cell is a PROJECTION refreshed from them — the same standing as +# a `metric` cell, and the same standing as the `posts` json window (which is even allowed to SHED +# posts to fit, precisely because the store still holds them). ⛔ A rollup may only ever READ. The +# moment one writes a number nothing else can re-derive, it has become a third copy. + +def _ut(): + """`core.user_tables`, imported LAZILY — and the laziness is measured, not stylistic. + + Pulling this module in at import time initialises the store layer earlier than + `automation_engine` used to, and the last time that happened it moved a store-commit COUNT + from three to four on an unrelated gate. The relational pass + needs three constants and two predicates from the field layer; it does not need to change + when this module is imported. + """ + import core.user_tables as _m + return _m + + +#: The fns by family — how a value is folded, and what a blank means in each. +_ROLLUP_NUM_FNS = frozenset({"sum", "average", "stdev", "min", "max"}) +_ROLLUP_BOOL_FNS = frozenset({"and", "or", "xor"}) +#: The count/order family — folded by their own arms above the lanes. +_ROLLUP_SEQ_FNS = frozenset({"countall", "counta", "count", "latest"}) +#: The text family. ⛔ IT IS A NAMED SET NOW AND IT USED TO BE THE FALL-THROUGH, which is how +#: wave 28 nearly shipped a wrong number that looked like data: `stdev` validated and STORED +#: (`core.user_tables.ROLLUP_FNS`) a commit before this function learned it, and an fn no arm +#: claims fell past both lanes into the join below — so a "Std deviation" column rendered +#: `"137684, 19561, 8123"`. Filled, plausible, and not a statistic. An unrecognised fn now +#: returns `""` ([[gate-answers-the-wrong-question]]: blank is the honest answer to a question +#: nothing can answer; a comma-joined list is a different question's answer wearing this label). +_ROLLUP_TEXT_FNS = frozenset({"concatenate", "arrayjoin", "arraycompact", "arrayunique"}) +#: ⭐⭐ WHAT THIS FOLD ACTUALLY IMPLEMENTS, DERIVED FROM THE ARMS RATHER THAN RESTATED. +#: `verify_automation` asserts this is IDENTICAL to `core.user_tables.ROLLUP_FNS` name for name +#: (contract C1's parity leg), so the validator can never again accept a function the fold cannot +#: compute — in EITHER direction. A hand-listed copy in the gate would have gone green on the +#: defect above, because the defect was that the two lists already disagreed. +ROLLUP_FOLD_FNS = frozenset(_ROLLUP_NUM_FNS | _ROLLUP_BOOL_FNS | _ROLLUP_SEQ_FNS + | _ROLLUP_TEXT_FNS) +_ROLLUP_TEXT_FNS = frozenset({"concatenate", "arrayjoin", "arraycompact", "arrayunique"}) +#: What `arrayjoin` puts between values. Airtable uses ", "; `concatenate` uses nothing. +_ROLLUP_JOIN = ", " +#: A rollup's own cell ceiling. The text fns can concatenate a whole column into one cell, and a +#: cell nobody can read is not an aggregate. +ROLLUP_MAX_CHARS = 4000 + + +def _sort_key(value, ftype): + """One cell → a sortable key, typed by the LINKED field's declared type. `None` when the cell + is blank or does not parse as its declared type. + + ⛔ TYPE-AWARE ON PURPOSE. `posted_at` is a `date` and `views` is an `int`; sorting either as a + string puts `2026-9-1` before `2026-10-1` and `9` after `100`. Since `sortBy` is what decides + WHICH rows a `limit` keeps, getting this wrong does not mis-order a display — it silently + averages the wrong twelve posts. + + ⛔⛔ **BLANKS ARE PARTITIONED OUT BY THE CALLER, NEVER RANKED BY A FLAG — AND THE FLAG VERSION + SHIPPED BROKEN FOR ONE DEPLOY.** This returned `(1, 0.0, "")` for a blank and `(0, key, "")` + otherwise, documented as "a blank sorts LAST under desc". It does the OPPOSITE: `reverse=True` + flips the whole tuple, so `(1, …)` sorted FIRST and an undated row displaced the most recent + real one out of the window. A sentinel inside the key cannot mean "last" in both directions, + because the direction is applied to the sentinel too. + ⚠ AND IT WAS NOT A CORNER CASE ON THE RUNG THAT MATTERS: `wave20-split` MEASURED `datetime` as + `None` on 24/24 posts from the Profiles dataset, and `_bd_post_identity` writes `posted_at` + only when the vendor sent one — so on a paid profile pull EVERY post row is blank here, every + key tied, and `limit 12` took whatever twelve came first in dict order. The e2e test passed + because its fixture posts all carried dates. + ⇒ `None` means "not comparable", the caller keeps those rows at the END whichever way it + sorts, and "an unknown date is not a recent one" is finally what the code does. + """ + raw = "" if value is None else str(value).strip() + if raw == "": + return None + if ftype in ("int", "currency", "pct", "rating"): + n = _lane_num(raw) + return (n,) if n is not None else None + if ftype == "date": + d = _parse_iso(raw.replace(" ", "T")) + return (d.timestamp(),) if d is not None else None + return (raw.lower(),) + + +def _sample_stdev(nums): + """SAMPLE standard deviation (n-1) of `nums`, or None under two values. + + ⭐ ONE IMPLEMENTATION, TWO READERS, and that is the point of lifting four lines into a + function: `_rollup_fold` RENDERS this number into a "Std deviation" column and + `_rollup_ref_threshold` COMPARES rows against it inside a `sigmas` condition. A second copy + would let a column and the filter beside it disagree about the same word on the same set — + the exact drift `top_up_views` was extracted to prevent one module over + ([[one-evaluator-per-question]]). + ⛔ None means UNANSWERABLE and no caller may read it as 0. + """ + if len(nums) < 2: + return None + mean = sum(nums) / len(nums) + return (sum((n - mean) ** 2 for n in nums) / (len(nums) - 1)) ** 0.5 + + +def _rollup_ref_threshold(rows, field, sigmas): + """`mean + sigmas*stdev` of `field` over `rows` — a statistic OF THE SCOPED SET (contract C1). + + ⛔ `rows` MUST already be the scoped window: ranked by `sortBy`, deduped, and cut by `limit`, + and NOT yet filtered by the conditions this threshold feeds. That order is the contract + (`core.user_tables.ROLLUP_REF_OPS`'s note says so in as many words) and both other orders + produce a plausible number: computing it before `limit` answers "2 sigma of everything this + account ever posted" under a column that says "of the last 10", and computing it after the + filter makes the threshold depend on the rows it is choosing — a definition that chases + itself. + + Returns None when the window cannot answer — fewer than two numeric values in `field`. + ⛔ THE CALLER MUST DROP THE ROW, NOT KEEP IT. "Beyond 2 sigma of one post" is not a question + with a permissive answer; letting an unanswerable leaf pass everything would silently turn + "the outliers" into "all of them", which is this module's worst failure mode wearing a filter. + """ + nums = [n for n in (_lane_num((r or {}).get(field)) for r in rows) if n is not None] + sd = _sample_stdev(nums) + if sd is None: + return None + return (sum(nums) / len(nums)) + float(sigmas) * sd + + +def _rollup_fold(fn, values, ftype="text"): + """`values` (raw cells, in the order the window kept them) → the aggregate, as a STRING. + + Returns `""` for "nothing to aggregate", NEVER `0`. ⛔ That distinction is this module's + oldest law and it bites hardest here: `sum` over no linked records is not zero, it is a + question with no rows to answer it, and a 0 in an "Avg views" column reads as a measurement + that the creator gets no views. + ⚠ The ONE exception is the count family, where zero IS the answer — "how many linked records" + over an empty set is genuinely 0, not unknown. + + `ftype` is the SOURCE column's DECLARED type on the linked table (2026-08-10, D-92). Only + `min`/`max` read it today; it defaults to `text` so every existing caller and every test that + folds a bare list keeps its exact previous answer. + """ + if fn == "countall": + return str(len(values)) + if fn == "latest": + # Ordering belongs to the rollup bag (`sortBy` is mandatory for this function). Preserve + # a blank on the newest row rather than reaching backwards and presenting an older value + # as current. + return "" if not values or values[0] is None else str(values[0])[:ROLLUP_MAX_CHARS] + if fn == "counta": + return str(len([v for v in values if str(v or "").strip() != ""])) + if fn == "count": + # Airtable's COUNT counts NUMERIC values; COUNTA counts non-empty ones. Keeping them + # distinct is the whole reason both exist. + return str(len([v for v in values if _lane_num(v) is not None])) + # ⭐⭐ 2026-08-10 — D-92 CLOSED: `min`/`max` OVER A DATE COLUMN. + # + # Both were numeric-only, so a rollup over `posted_at`, `due_date` or `order_date` rendered + # BLANK forever while looking completely configured — the failure this module refuses + # everywhere else, arriving through the one fold that had no type awareness. "Earliest order" + # and "latest invoice due" are the two most ordinary date rollups there are, and + # `odoo_relational` already ships `latest` over `due_date`, so the vocabulary claimed dates + # and the fold did not. + # + # ⛔ NOT FIXED BY SNIFFING THE VALUES. An ISO date sorts correctly as a string, so a + # string-compare fallback would have worked on well-formed data and silently mis-ordered a + # `Aug 5, 2026` or a `2026-9-1` — [[measure-the-real-call]]'s shape. The DECLARED type is + # already at this call site (`linked_types`), and `_sort_key` is already the one function that + # turns a typed cell into a comparable key, blanks partitioned out. This reuses both rather + # than growing a second idea of what a date is. + # ⚠ RETURNS THE CELL, NOT THE KEY. `_sort_key` yields a comparison tuple; the answer a person + # wants in the column is the stored date string exactly as the source row spells it. + # ⚠ `min`/`max` ONLY. `sum`/`average`/`stdev` over dates are not blank by oversight — the mean + # of two timestamps is a number this product has no column type for, and inventing one here + # would be a value with no author. + if fn in ("min", "max") and ftype == "date": + keyed = [(k, str(v)) for k, v in + ((_sort_key(v, "date"), v) for v in values) if k is not None] + if not keyed: + return "" + return (min(keyed) if fn == "min" else max(keyed))[1][:ROLLUP_MAX_CHARS] + if fn in _ROLLUP_NUM_FNS: + nums = [n for n in (_lane_num(v) for v in values) if n is not None] + if not nums: + return "" + if fn == "stdev": + # ⭐⭐ SAMPLE standard deviation (n-1), and the divisor is a ruling, not a preference + # (C1 / `core.user_tables.ROLLUP_FNS`'s note): a rollup folds the rows that happen to + # be LINKED, which is a sample of an account's posting history and not its entirety. + # ⚠ Fixture to check a refactor against: [2,4,4,4,5,5,7,9] -> 2.14. The POPULATION + # form gives 2.00 on the same input, so a test that ever reads 2.00 has silently + # switched divisors. + # ⛔ FEWER THAN TWO VALUES IS "" AND NEVER "0". n-1 = 0 would divide by zero, but the + # honest reason is upstream of the arithmetic: one measurement has no spread to + # report, and a 0 in a "Std deviation" column reads as PERFECT CONSISTENCY — the + # single most confident thing this column can say, asserted from a single row. Same + # law as the blank `sum`, and it bites harder here. + out = _sample_stdev(nums) + if out is None: + return "" + else: + out = (sum(nums) if fn == "sum" else min(nums) if fn == "min" + else max(nums) if fn == "max" else sum(nums) / len(nums)) + return f"{out:.0f}" if float(out).is_integer() else f"{out:.2f}" + if fn in _ROLLUP_BOOL_FNS: + # A checkbox cell is '1'/'' in this product, so truth is "non-blank and not a zero". + flags = [str(v or "").strip() not in ("", "0", "false", "False") for v in values] + if not flags: + return "" + hit = (all(flags) if fn == "and" else any(flags) if fn == "or" + else sum(1 for f in flags if f) % 2 == 1) + return "1" if hit else "" + # --- the text family. ⛔ CLAIMED BY NAME, NEVER BY FALL-THROUGH — see `_ROLLUP_TEXT_FNS`. + # An fn no arm above recognises returns "" rather than a comma-joined dump of every value, + # which is the shape a not-yet-implemented aggregate wore for one commit of wave 28. + if fn not in _ROLLUP_TEXT_FNS: + return "" + vals = [str(v).strip() for v in values if str(v or "").strip() != ""] + if fn == "arrayunique": + seen, uniq = set(), [] + for v in vals: + if v.lower() not in seen: + seen.add(v.lower()) + uniq.append(v) + vals = uniq + if not vals: + return "" + text = ("".join(vals) if fn == "concatenate" else _ROLLUP_JOIN.join(vals)) + return text[:ROLLUP_MAX_CHARS] + + +def _link_from_key(fields, bag): + """Which column on THIS table supplies the join value. + + Declared `from` wins; otherwise the PROFILE-flagged column, then the PINNED one. ⭐ That + fallback chain is what makes an Instagram database link up with no configuration at all — + the "automatically" in the owner's instruction — and it is the same chain the grid already + uses to decide a table's identity column, rather than a second opinion about it. + """ + declared = str((bag or {}).get("from") or "").strip() + if declared: + return declared + prof = next((f for f in fields if isinstance(f.get("profile"), dict)), None) + if prof: + return str(prof.get("key") or "") + pin = next((f for f in fields if f.get("pinned") is True), None) + return str(pin.get("key") or "") if pin else "" + + +def _linked_rows_by_join(linked, on_key): + """`{join value (lower-cased) -> [(row_id, row)]}` over one linked table, built ONCE. + + ⚠ Lower-cased because the join values this exists for are Instagram handles, which the + profile flag already normalises to lower case on one side and which a hand-typed cell on the + other side may not. A join that misses on case is a relation that silently reports zero. + """ + idx = {} + for rid, row in ((linked or {}).get("rows") or {}).items(): + k = str((row or {}).get(on_key) or "").strip().lower() + if k: + idx.setdefault(k, []).append((str(rid), row or {})) + return idx + + +def _rollup_condition_matches(row, condition, field_types, ref_value=None): + """Evaluate one Airtable-style linked-record condition against a candidate row. + + `ref_value` is the threshold a `ref: {sigmas}` leaf compares against, already computed by the + caller over the SCOPED set (`_rollup_ref_threshold`). ⛔ It is passed IN rather than computed + here because this function sees one row and the statistic is a property of the whole window — + a version that reached for the set from inside would be recomputing the same mean once per + row, and would have to be handed the window anyway. + ⚠ `None` means the window could not answer, and the leaf then matches NOTHING. See the + threshold helper for why the permissive reading is the dangerous one. + """ + field = str((condition or {}).get("field") or "") + op = str((condition or {}).get("op") or "") + raw = (row or {}).get(field) + text = str(raw or "").strip() + if op == "is_empty": + return text == "" + if op == "is_not_empty": + return text != "" + if (condition or {}).get("ref") is not None: + # ⛔ NUMERIC LANE ONLY, BOTH SIDES. The validator already restricts `ref` to the ordering + # ops, and a row whose cell is blank or unparseable has no position relative to a computed + # threshold — it is not "below" it. Dropping it is the same partition law `_sort_key` + # follows: unrankable is not a rank ([[sentinel-in-a-sort-key]]). + left_num = _lane_num(text) + if ref_value is None or left_num is None: + return False + return ((op == "gt" and left_num > ref_value) + or (op == "gte" and left_num >= ref_value) + or (op == "lt" and left_num < ref_value) + or (op == "lte" and left_num <= ref_value)) + wanted = str((condition or {}).get("value") or "").strip() + if op == "contains": + return wanted.casefold() in text.casefold() + if op == "not_contains": + return wanted.casefold() not in text.casefold() + if op in ("eq", "neq"): + left_num, right_num = _lane_num(text), _lane_num(wanted) + equal = (left_num == right_num if left_num is not None and right_num is not None + else text.casefold() == wanted.casefold()) + return equal if op == "eq" else not equal + ftype = (field_types or {}).get(field, "text") + left = _sort_key(text, ftype) + right = _sort_key(wanted, ftype) + if left is None or right is None: + return False + return ((op == "gt" and left > right) or (op == "gte" and left >= right) + or (op == "lt" and left < right) or (op == "lte" and left <= right)) + + +def compute_relation_cells(rt, table_key, tables=None): + """Recompute every DERIVED LINK cell and every ROLLUP cell on ONE table. Returns rows touched. + + Zero store reads when the table declares neither kind — the same cheap-by-construction shape + `compute_metric_cells` has, so walking every table on a tick costs a dict scan per table. + """ + store = tables if tables is not None else ut_all(rt) + t = (store or {}).get(table_key) + fields = list((t or {}).get("fields") or []) + links = [f for f in fields if _ut().is_derived_link(f)] + rollups = [f for f in fields if isinstance(f.get("rollup"), dict)] + if not links and not rollups: + return 0 + rows = (t or {}).get("rows") or {} + by_key = {str(f.get("key")): f for f in fields} + + # --- resolve every link field ONCE per table, not once per row. + # `resolved[link_key][row_id] = [(linked_row_id, linked_row), ...]` + resolved, linked_types = {}, {} + for f in links + [by_key.get(str((r.get("rollup") or {}).get("link"))) for r in rollups]: + lk_key = str((f or {}).get("key") or "") + if not lk_key or lk_key in resolved or not isinstance((f or {}).get("link"), dict): + continue + bag = f["link"] + linked = (store or {}).get(str(bag.get("table") or "")) or {} + linked_types[lk_key] = {str(lf.get("key")): str(lf.get("type") or "text") + for lf in (linked.get("fields") or [])} + if bag.get("inverse"): + # Airtable's reciprocal side: this row is linked to every SOURCE row whose ordinary + # link cell contains this row id. The source cell remains the one relationship truth. + source_rows = linked.get("rows") or {} + inverse_key = str(bag.get("inverse") or "") + inverse_index = {} + for source_id, source_row in source_rows.items(): + for target_id in [part.strip() for part in + str((source_row or {}).get(inverse_key) or "").split(",")]: + if target_id: + inverse_index.setdefault(target_id, []).append( + (str(source_id), source_row or {})) + resolved[lk_key] = {str(rid): inverse_index.get(str(rid), []) for rid in rows} + elif bag.get("on"): + idx = _linked_rows_by_join(linked, str(bag["on"])) + from_key = _link_from_key(fields, bag) + resolved[lk_key] = { + str(rid): idx.get(str((row or {}).get(from_key) or "").strip().lower(), []) + for rid, row in rows.items()} if from_key else {} + else: + # An ORDINARY link: the cell IS the relation. Ids the linked table no longer holds + # are dropped rather than carried — a link to a deleted row is not a link. + lrows = (linked.get("rows") or {}) + resolved[lk_key] = { + str(rid): [(i, lrows[i]) for i in + [s.strip() for s in str((row or {}).get(lk_key) or "").split(",")] + if i and i in lrows] + for rid, row in rows.items()} + + changes = {} + for rid, row in rows.items(): + rid = str(rid) + row = row or {} + for f in links: + fk = str(f["key"]) + hits = (resolved.get(fk) or {}).get(rid) or [] + if f["link"].get("single"): + hits = hits[:1] + # ⛔ THE CAP IS A DISPLAY CAP AND THE ROLLUPS DO NOT READ THROUGH IT. A derived cell + # is a projection of `resolved`, which is uncapped and is what every rollup below + # consumes — so a profile with 900 posts shows the first 500 ids and still averages + # over all 900. Same argument the `posts` window already makes: shedding is safe + # precisely because the authoritative store still holds everything. + want = ",".join(i for i, _r in hits[:_ut().LINK_MAX_IDS]) + if str(row.get(fk, "")) != want: + changes.setdefault(rid, {})[fk] = want + for f in rollups: + fk, bag = str(f["key"]), f["rollup"] + lk_key = str(bag.get("link") or "") + hits = list((resolved.get(lk_key) or {}).get(rid) or []) + # ⚠ A rollup whose link field does not exist (renamed, deleted) resolves to NOTHING + # and therefore to a blank cell — never to a stale number. A column that keeps + # printing yesterday's answer after its input is gone is the worst of the options. + # ⭐⭐ WAVE 28 / CONTRACT C1 — SCOPE FIRST, FILTER SECOND, AND THE TWO USED TO BE THE + # OTHER WAY ROUND. The conditions block stood HERE, above the ranking, so + # "last 10 posts where views > X" meant *the 10 most recent of the posts over X* + # rather than *the ones over X among the last 10* — two different windows wearing one + # sentence. Harmless while a threshold was a literal; incoherent the moment a + # threshold is a statistic OF the window, because the set being described and the set + # doing the describing would be different sets. + # ⛔ THIS ORDER IS THE CONTRACT, not an implementation choice: `core.user_tables`'s + # `ROLLUP_REF_OPS` note states it ("the scope picks the window, THEN the threshold is + # computed over that window, THEN the conditions filter it") and the validator half + # was written against it. + # ⚠ IT IS A BEHAVIOUR CHANGE FOR EXACTLY ONE SHAPE: a stored rollup carrying BOTH + # `conditions` AND `limit`. No shipped preset does (measured across `odoo_relational` + # and the IG presets — the one preset with conditions, `_OPEN_ONLY`, is a `countall` + # with no limit), so the blast radius is user-built rollups only. + # ⭐⭐ 2026-08-09 (owner) — THE PRE-FILTER, ABOVE THE RANKING. Owner: *"instead of last + # 12 posts, we also want to make it so its last N record, where the record's Status is + # video."* `conditions` cannot answer that: C1 moved them BELOW the window on purpose, + # so they select among the rows the window already kept. `where` selects WHICH rows + # the window is spent on. + # ⛔ ABOVE `distinctBy` TOO, not merely above the sort. Dedup keeps the first row per + # identity; run it first and a carousel could claim the slot its reel sibling needed, + # so the window would come up short for a reason nothing on screen explains. + # ⚠ NO `ref` REACHES HERE — `_clean_rollup` refuses a set-statistic threshold in this + # list, because at this point there is no fixed set for a statistic to be about. + where = list(bag.get("where") or []) + if where: + where_matches = lambda pair: [ + _rollup_condition_matches(pair[1], condition, + linked_types.get(lk_key) or {}, ref_value=None) + for condition in where] + if str(bag.get("whereConj") or "and") == "or": + hits = [pair for pair in hits if any(where_matches(pair))] + else: + hits = [pair for pair in hits if all(where_matches(pair))] + sort_by = str(bag.get("sortBy") or "") + if sort_by: + ftype = (linked_types.get(lk_key) or {}).get(sort_by, "text") + # ⛔ PARTITION, THEN SORT. A row whose sort cell is blank or unparseable is not + # rankable, and it must land at the END whichever direction is asked for — which + # a sentinel inside the sort key cannot do, because `reverse` flips the sentinel + # too (see `_sort_key`). Unrankable rows are appended, so a `limit` spends its + # window on rows that HAVE the value before it falls back to ones that do not. + keyed = [(pair, _sort_key(pair[1].get(sort_by), ftype)) for pair in hits] + rankable = [(p, k) for p, k in keyed if k is not None] + rankable.sort(key=lambda pk: pk[1], + reverse=str(bag.get("sortDir") or "desc") == "desc") + hits = [p for p, _k in rankable] + [p for p, k in keyed if k is None] + distinct_by = str(bag.get("distinctBy") or "") + if distinct_by: + seen, unique = set(), [] + for pair in hits: + identity = str(pair[1].get(distinct_by) or "").strip().lower() + # A blank is not an identity. Keep it rather than collapsing every unknown + # record into one synthetic duplicate. + if identity and identity in seen: + continue + if identity: + seen.add(identity) + unique.append(pair) + hits = unique + limit = int(bag.get("limit") or 0) + if limit: + hits = hits[:limit] + # --- the window is now FIXED, so a set-statistic threshold has a set to be about. + conditions = list(bag.get("conditions") or []) + if conditions: + # ⚠ RESOLVED ONCE PER LEAF, NOT ONCE PER ROW. The threshold is a property of the + # window; computing it inside `matches` would recompute the same mean for every + # candidate and — worse — would invite computing it over a set that the filter is + # already shrinking underneath it. + # ⚠ `.get("sigmas", 0.0)`, never `... or 0.0` — a legitimate `sigmas: 0` ("beyond + # the mean") is falsy, and the `or` spelling would silently rewrite it to the same + # number by accident. It reads identically and is right for the wrong reason, + # which is how it survives a review. + refs = [_rollup_ref_threshold( + [r for _i, r in hits], str(c.get("field") or ""), + (c.get("ref") or {}).get("sigmas", 0.0)) + if isinstance(c, dict) and c.get("ref") is not None else None + for c in conditions] + matches = lambda pair: [ + _rollup_condition_matches(pair[1], condition, + linked_types.get(lk_key) or {}, ref_value=ref) + for condition, ref in zip(conditions, refs)] + if str(bag.get("conditionConj") or "and") == "or": + hits = [pair for pair in hits if any(matches(pair))] + else: + hits = [pair for pair in hits if all(matches(pair))] + src = str(bag.get("field") or "") + # D-92 — the SOURCE column's declared type, read from the same `linked_types` map the + # sort path above uses. `countall` folds a synthetic `[1]*n` with no source column at + # all, so the default stands for it. + want = _rollup_fold(str(bag.get("fn") or ""), + [r.get(src) for _i, r in hits] if src else [1] * len(hits), + (linked_types.get(lk_key) or {}).get(src, "text")) + if str(row.get(fk, "")) != want: + changes.setdefault(rid, {})[fk] = want + if not changes: + return 0 + + # A run that just wrote linked rows passes its in-flight user_tables bucket here so the + # relation refresh joins the SAME coalesced commit. The standalone/tick path below keeps + # the public helper's old persist-on-change behaviour. + if tables is not None: + tt = tables.get(table_key) + if tt is not None: + for r, vals in changes.items(): + tt.setdefault("rows", {}).setdefault(r, {}).update(vals) + return len(changes) + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + tt = cur.get(table_key) + if tt is not None: + for r, vals in changes.items(): + tt.setdefault("rows", {}).setdefault(r, {}).update(vals) + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") + return len(changes) + + +def _refresh_relations_inplace(tables, log=print): + """Refresh every relation against one mutable user_tables bucket; perform no store write.""" + touched = 0 + for tk, table in list((tables or {}).items()): + fields = (table or {}).get("fields") or [] + if not any(f.get("type") == "rollup" or _ut().is_derived_link(f) for f in fields): + continue + try: + touched += compute_relation_cells(None, tk, tables=tables) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] relation refresh {tk} failed: {type(e).__name__}: {e}") + return touched + + +def refresh_relations(rt, log=print): + """The tick half of the relational pass — the twin of `refresh_metrics`. + + ⚠ Runs for EVERY table, because a link can point anywhere: a rollup on table A goes stale + when table B gains a row, and A has no way to know that happened. Cheap by construction — a + table declaring neither kind costs one dict scan. + """ + snapshot = { + str(key): {**(table or {}), + "rows": {str(rid): dict(row or {}) + for rid, row in ((table or {}).get("rows") or {}).items()}} + for key, table in (ut_all(rt) or {}).items() + } + if not _refresh_relations_inplace(snapshot, log=log): + return 0 + actual = [0] + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + actual[0] = _refresh_relations_inplace(cur, log=log) + return cur + + # ⭐ ASYNC, and for TWO independent reasons (2026-08-09, the lost-record bug). + # + # 1. COST, which was wave 27 item 2's whole point: this pass runs after every row write, and + # `flush="sync"` made each one a blocking HF commit against the 256-commits/hr budget. + # A derived-cell recompute has no business forcing a commit on someone typing. + # 2. It was the deterministic TRIGGER of the bug: `add_row` writes `flush='async'`, so a new + # row lives only in the store cache for 2-20s, and this call — on the SAME key + # (`UT_STORE_KEY == user_tables`) — used to `_read_strict` that cache away and upload the + # result. `POST /rows` answered 201 and the row was gone. + # + # ⚠ THE ROOT FIX IS IN `core/store.py` (a sync RMW no longer discards a dirty cache) and it + # is what makes the other ~39 sync writers of this key safe. This line is not that fix and + # must not be mistaken for it — it removes the trigger and the cost, nothing more. Both + # landed together on purpose: one is correctness, one is the hot path. + rt.update(UT_STORE_KEY, _up, flush="async") + return actual[0] + + +def refresh_metrics(rt, today=None, log=print): + """The tick half of the C7 amendment: `today` advances at tick cadence, so a date-window + metric can never go staler than one tick while a scheduler exists. Cheap by construction — + a table without metric fields costs a dict scan and nothing else.""" + touched = 0 + for tk, t in ut_all(rt).items(): + if metric_fields_of_table(t): + try: + touched += compute_metric_cells(rt, tk, today=today) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] metric refresh {tk} failed: {type(e).__name__}: {e}") + return touched + + +def purge_subject(rt, handle): + """D-24: right-to-erasure for ONE Instagram subject — every row about them leaves the + tenant's four `ut_ig_*` tables AND the platform master (R2 made the master half + non-optional: a purge that missed the pooled copy would not be erasure). Returns + `{table: removed}` counts, master rows prefixed `master:` — every count drills to what is + now ABSENT, which is the one aggregate whose drill is emptiness.""" + subject = str(handle or "").strip().lstrip("@").lower() + if not subject: + return {} + counts = {} + tables = ut_all(rt) + post_rows = (tables.get("ut_ig_posts") or {}).get("rows") or {} + codes = {str(r.get("shortcode") or "") for r in post_rows.values() + if str((r or {}).get("influencer_key") or "").strip().lower() == subject} + + keeps = { + "ut_ig_snapshots": lambda r: str((r or {}).get("influencer_key") + or "").strip().lower() != subject, + "ut_ig_posts": lambda r: str((r or {}).get("influencer_key") + or "").strip().lower() != subject, + "ut_ig_post_snapshots": lambda r: str((r or {}).get("shortcode") or "") not in codes, + DISCOVER_TABLE: lambda r: str((r or {}).get("handle") + or "").strip().lower() != subject, + } + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + for tk, keep in keeps.items(): + t = cur.get(tk) + if t is None: + continue + rows = t.get("rows") or {} + nxt = {rid: r for rid, r in rows.items() if keep(r)} + counts[tk] = len(rows) - len(nxt) + t["rows"] = nxt + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") + import ig_master + for bucket, n in (ig_master.purge_handle(subject) or {}).items(): + counts[f"master:{bucket}"] = n + return counts + + +# --------------------------------------------------------------------------------------------- +# TRIGGERS (wave 22, contract C3 + amendment A2 — owner ruling R4; closes D-33) +# --------------------------------------------------------------------------------------------- +# Six ways an automation starts, exactly: manual | schedule | event_field | record_created | +# webhook | email. The first two are what always existed (Run now; cron via the tick). The four +# new ones are EVENTS, and A2 makes their discipline LAW rather than taste: +# +# * **EDGE, NEVER LEVEL (A2(1)).** A condition trigger fires on entering the matching state, +# not for being in it. Implemented as per-record ARMED state over successive evaluations +# (`state.eventDisarmed`): a record fires when it matches while armed, DISARMS, and re-arms +# only by evaluating False — Airtable's documented leave-and-re-enter rule, without needing +# a before-image of a row whose truth is spread over strata. Enabling a trigger SEEDS the +# disarmed set with everything currently matching, so already-matching records do not fire +# (`_seed_event_state`). A settle window coalesces write bursts (the per-keystroke scar). +# * **LOOP PREVENTION IS STRUCTURAL (A2(2)).** The hooks live on the HUMAN doors only +# (`grid_events.overlay_patch`, `user_tables.add_row`); the engine's own writers +# (`ut_write_rows`, the runners' coalesced updates, `patch_cells` from `move_card`) never +# emit — so an automation's write cannot fire event triggers, its own or a sibling's, by +# construction. The circuit breaker on top (>60 fires/5 min auto-pauses with the reason as +# a statusNote) catches whatever construction did not foresee. +# * **FLOOD HOLD (A2(3)).** One evaluation yielding more than 100 candidate records holds +# instead of running — a partial run entry names the count and the deliberate way through +# (Run now). C4's discovery guard is this rule's special case. +# * **REFIRE DEFAULTS (A2(4)), hard-coded this wave:** record_created fires once per record +# EVER (a high-water mark over row ids, so an undo-restored row cannot re-fire); +# event_field fires every transition. + +# ── WAVE 23 · C3 — the trigger vocabulary v2 (owner ruling R2). ─────────────────────────────── +# Airtable's phrasing, because the owner asked for Airtable's builder and a trigger list that +# renames the same events is a second vocabulary to learn for no gain. +# +# ⚠ `event_field` KEPT ITS KEY and changed its LABEL to "When a record matches conditions". +# Renaming the key would have orphaned every stored trigger in production for a caption; the key +# is the contract with the store, the label is the contract with the reader, and they are allowed +# to disagree. What genuinely widened is its SHAPE: the watched field is now OPTIONAL, so the +# trigger covers Airtable's condition-only form (any write to the table, evaluated against a C4 +# tree) as well as wave 22's watch-one-field form. Both are the same edge rule underneath. +# +# ⛔ PLANNED ≠ STORABLE. `button_clicked` / `comment_added` ride the wire so the picker can show +# them faded with a reason (R2: "never a dead control") — and `clean_trigger` REFUSES them with a +# sentence. A vocabulary that renders an option the validator rejects is the wave-9 silent-drop +# class wearing a friendlier face; here the two lists are separate on purpose and the refusal +# names the state rather than pretending the key is unknown. +# ── WAVE 24 · C-TRIG (owner ruling R6). ─────────────────────────────────────────────────────── +# ⭐ INSTAGRAM DISCOVERY BECOMES A TRIGGER. It was a KIND you picked in a create wizard; the +# wizard is deleted, and "when an Instagram profile fits a criteria" is the honest shape anyway — +# it is the event this flow starts from. Picking it sets the definition's kind to +# `discover_instagram` (law 1), which is the ONLY way that kind is reachable now. +# +# ⛔ IT IS NOT A TABLE TRIGGER AND NOT A ROW TRIGGER. It watches nothing: it MAKES rows, on the +# schedule (or on Run now), so it stays out of both lists below and its node switch flips the +# CRON — see `TRIGGER_SCHEDULE_KEYS`. +# +# ⭐⭐ WAVE 29 (item 7 · D-9 · R1) — `tiktok_profile_match` MOVED HERE FROM `TRIGGER_PLANNED`, and +# that move is the whole of "TikTok is a real trigger now". The label was written a wave early in +# the owner's own words and has not changed; what changed is which tuple it sits in, because +# `clean_trigger` refuses the planned list with a sentence and accepts this one. +TRIGGER_KEYS = ("manual", "schedule", "event_field", "record_updated", "record_created", + "enters_view", "webhook", "email", "form_submitted", "ig_profile_match", + "tiktok_profile_match") +#: ⭐ WAVE 25 · C2 / owner ruling R9 — `web_page_changed` JOINS THE PLANNED LIST, and joining THIS +#: tuple rather than `TRIGGER_KEYS` is the whole of its implementation. `clean_trigger` refuses +#: everything here with a sentence, so the faded row is a wall; the picker shows it so the Scraper +#: section is not a section of one. +#: ⚠ `web_page_changed` IS NOT THE WEB ACTION (D-51). A trigger that notices a page changed and an +#: action that drives a browser are different builds; this row must not be read as progress on D-51. +TRIGGER_PLANNED = ("button_clicked", "comment_added", "web_page_changed") +TRIGGER_LABELS = { + "manual": "Manual", + "schedule": "At a scheduled time", + "event_field": "When a record matches conditions", + "record_updated": "When a record is updated", + "record_created": "When a record is created", + "enters_view": "When a record enters a view", + "webhook": "When a webhook is received", + "email": "When an email arrives", + "form_submitted": "When a form is submitted", + "ig_profile_match": "When an Instagram profile fits a criteria", + "button_clicked": "When a button is clicked", + "comment_added": "When a comment is added", + "web_page_changed": "When a website page changes", + "tiktok_profile_match": "When a TikTok profile fits a criteria", +} + +# ── WAVE 25 · C2 — THE PICKER TAXONOMY, and it lives HERE beside the vocabulary it describes. ── +# `group` already rode the wire as "Standard"/"Sources" (`routes_automation`), which is a +# distinction about where a trigger came FROM rather than about what a person is choosing. The +# question the picker actually asks is: does this fire on TIME, on your own DATA, or because +# something OUTSIDE said so. Three answers, and every trigger has exactly one. +# +# ⛔ TWO CONTROLS, NOT ONE, AND THE FIRST DRAFT HAD ONLY THE WRONG HALF. Indexing this map +# directly (`TRIGGER_GROUP_OF[k]`) makes an unclassified trigger a KeyError — which is exactly the +# incident `_triggers_vocab`'s `per.get(k, ...)` comment records: a key added to `TRIGGER_KEYS` +# without remembering a dict beside it 500'd `GET /automations`, the payload the whole automation +# surface polls every 2.5 s, with every gate green. A mis-grouped row is a cosmetic bug; a 500 is +# the surface. So: +# * RUNTIME fails SOFT — an unclassified trigger falls into `other`, which sorts LAST (the rule +# `ACTION_GROUP_ORDER` already uses: an unordered group sorts last, never first, because +# appearing at the top looks deliberate) and is honestly captioned rather than smuggled into +# Database. +# * THE GATE fails HARD — `verify_automation` asserts that NO shipped trigger lands in `other`, +# so the fallback is provably dead code in production and the classification is still +# mandatory. The fallback catches the accident; the gate stops it shipping. +# ⭐ WAVE 34 · R19 — CONNECTOR SITS ABOVE DATABASE, DIRECTLY UNDER TIME. Owner, verbatim: +# *"In the Trigger picker, the Connector section sits directly above the Database section, just +# under the Time trigger types."* So the two orders below are SWAPPED against wave 24's, and +# nothing else moved: same keys, same labels, same fallback. +# ⛔ THIS IS THE WHOLE OF R19 AND IT IS DELIBERATELY NOT A CLIENT CHANGE. `steps.ts::groupTriggers` +# sorts by each option's `groupOrder` and by nothing else, so re-ordering an array on the client +# would look right in a fixture and be wrong in production the moment the server re-sorted. +# ⚠ `verify_steps.py` CANNOT WITNESS THIS EDIT: its C2 leg supplies its OWN `groupOrder` in a +# fixture and asserts the client honours it, which stays true whatever these numbers say. The +# check that binds R19 to this table lives in `verify_automation` beside the vocab section. +TRIGGER_GROUPS = {"time": {"label": "Time", "order": 1}, + "connector": {"label": "Connector", "order": 2}, + "database": {"label": "Database", "order": 3}, + "other": {"label": "Other", "order": 99}} +#: Where an unclassified trigger goes. ⚠ Reaching this in production is a BUG the gate exists to +#: prevent — it is the soft landing, not a category anybody should be adding triggers to. +TRIGGER_GROUP_FALLBACK = "other" +TRIGGER_GROUP_OF = { + "manual": "time", "schedule": "time", + "event_field": "database", "record_updated": "database", "record_created": "database", + "enters_view": "database", "form_submitted": "database", + "button_clicked": "database", "comment_added": "database", + "email": "connector", "webhook": "connector", "ig_profile_match": "connector", + "web_page_changed": "connector", "tiktok_profile_match": "connector", +} +#: The SUB-group inside "Connector" — which connected thing this trigger comes through. +#: ⚠ THESE KEYS ARE GROUPING HANDLES FOR THE PICKER, NOT connector-directory slugs, and the two +#: genuinely differ: the directory's OAuth row for Gmail is `google` (the provider), while a +#: person choosing a trigger is picking *Gmail* (the product). A client that joined this key +#: against `/connectors/directory` would match `scraper` and `webhooks` and miss `gmail` — so it +#: must group by it and render `label`, never look it up. Said here because the miss would be +#: silent and partial, which is the worst shape. +#: (⚠ that example USED to read "`scraper` and `tiktok`" — R3 retired `tiktok` as a handle, and +#: the sentence is corrected here rather than left to rot into a lie about a key that is gone.) +TRIGGER_CONNECTOR = { + "email": {"key": "gmail", "label": "Gmail"}, + "webhook": {"key": "webhooks", "label": "Webhooks"}, + # ⭐ WAVE 30 · R3 — ONE "SCRAPER" BUCKET, AND IT HOLDS BOTH PLATFORMS. + # The owner, verbatim and for the third wave running: *"I say this multiple times already the + # damn Tiktok and Instagram belongs in the same bucket when creating the automation its under + # Scraper … Only when I click 'Scraper' under each automation trigger and actions would I see + # the option to choose either Instagram OR TikTok. That's it."* + # + # ⛔ THIS SUPERSEDES WAVE 29's RULE, and the old rule was not a typo — it was an argument: + # *"TikTok is its own connector, not the Scraper's … because the sub-group answers WHICH + # PRODUCT and never HOW BUILT."* Coherent, and not what was asked for. Instagram and TikTok are + # two PRODUCTS of one CAPABILITY (a social scraper bought from one vendor); a person opening + # this picker is choosing the capability first and the platform second. `verify_automation` + # asserted the old rule as an assertion AND as prose — a shipped gate forbidding the owner's + # ruling is most of why this complaint survived two waves — so it is INVERTED in this same + # change, comment included. + # + # ⚠ The Scraper sub-group now holds THREE rows: two built (Instagram, TikTok) and one faded + # (the page-change trigger). No display ORDER is emitted here — the client groups on `key` and + # owns its own ordering (contract C1). `tiktok` ceases to exist as a grouping handle. + "ig_profile_match": {"key": "scraper", "label": "Scraper"}, + "web_page_changed": {"key": "scraper", "label": "Scraper"}, + "tiktok_profile_match": {"key": "scraper", "label": "Scraper"}, +} +#: Triggers that watch a database and therefore need one named before they can fire. +TRIGGER_TABLE_KEYS = ("event_field", "record_updated", "record_created", "enters_view", + "form_submitted") +#: ⭐ WAVE 24 — triggers whose NODE SWITCH means the CRON rather than the trigger itself. +#: `manual`/`schedule` are not stored at all; `ig_profile_match` is stored and IS schedule-driven, +#: so flipping its node must flip the schedule. +#: +#: ⚠ THIS REPLACES A HAND-LISTED TUPLE IN `toggle_node` THAT WAS ALREADY WRONG. It read +#: `("event_field", "record_created", "webhook", "email")` — omitting `record_updated`, +#: `enters_view` and `form_submitted`, all three of which have been storable since wave 23. For +#: those, clicking the trigger node's switch flipped the CRON under a node labelled "When a +#: record is updated": a switch that lies, which is exactly what the tuple at `graph()` warns +#: about eight lines into its own comment. Derived from one named set now, so a trigger added to +#: `TRIGGER_KEYS` cannot silently join the wrong side of it. +#: ⚠ WAVE 29 — `tiktok_profile_match` BELONGS HERE FOR THE SAME REASON `ig_profile_match` DOES, +#: and forgetting it is precisely the failure this constant's own note describes: it watches no +#: table and MAKES rows on the schedule, so its node switch has nothing to flip but the cron. Left +#: out, a person clicking the TikTok trigger node's switch would toggle the trigger itself while +#: the schedule kept firing — a switch that lies. +TRIGGER_SCHEDULE_KEYS = ("manual", "schedule", "ig_profile_match", "tiktok_profile_match") +#: ⭐ WAVE 25 — DEBT D-55: "the cron drives this one", on the wire at last. +#: +#: ⛔ `TRIGGER_SCHEDULE_KEYS` MUST NOT SHIP VERBATIM, and the one-element difference is the entire +#: reason this constant exists rather than the tuple above being sent. That set answers "which +#: way does this trigger's NODE SWITCH flip" — and `manual` is in it only because a manual +#: automation's switch has nothing else to flip. Shipping it as "the cron drives this" would draw +#: a schedule face on the one trigger whose whole sentence is "It runs only when you press Run +#: now": a control contradicting its own description. +#: +#: D-55's history is why it is DERIVED rather than listed: the client carried +#: `CRON_DRIVEN_TRIGGERS = ["schedule", "ig_profile_match"]` — a hand-kept copy of a server fact +#: that fails VISIBLY but silently (a new cron-driven trigger simply shows no schedule face). +#: Subtracting from the engine's own set means a trigger added there cannot be forgotten here. +TRIGGER_CRON_KEYS = frozenset(TRIGGER_SCHEDULE_KEYS) - {"manual"} +#: Triggers the ROW HOOKS drive (as opposed to the tick, or an inbound HTTP call). Named once so +#: `grid_hook` and the gates read the same list instead of two matching `in (...)` tuples. +TRIGGER_ROW_KEYS = ("event_field", "record_updated", "record_created", "enters_view") +MAX_WATCH_FIELDS = 12 +#: The settle window for field-change bursts (A2(1)). 0 evaluates INLINE — the gates run there, +#: and so would a deployment that prefers immediacy over coalescing. +EVENT_SETTLE_SECONDS = float(os.environ.get("AIOS_EVENT_SETTLE_SECONDS") or 15) +FIRE_LIMIT = 60 # A2(2): fires per window before the breaker pauses +FIRE_WINDOW_SECONDS = 300 +FLOOD_LIMIT = 100 # A2(3): candidate records one evaluation may act on +EMAIL_SEEN_CAP = 500 # message-id dedupe memory per automation +EMAIL_MAX_PER_POLL = 25 # bounded by construction — a poll is a tick guest +CONSECUTIVE_FAILURE_PAUSE = 5 # airtable-brief rec 6: a dead credential must not burn quota + +EMAIL_FIELDS = [ + field_def("email_id", "Email id"), field_def("email_from", "From"), + field_def("email_subject", "Subject"), field_def("email_date", "Date"), + field_def("email_snippet", "Snippet"), field_def("email_seen_at", "Seen at"), +] + + +def clean_trigger(raw, previous=None): + """Validate a definition's `trigger`. Returns `(trigger|None, error)` — None is legal and + means what it always meant: manual + whatever `schedule` says. + + ⚠ A3 (2026-08-05): the stored/wire name is `key` (`kind` accepted on input for symmetry + with the definition's own vocabulary). And an INCOMPLETE event trigger is STORED INERT + rather than refused — the picker writes `{key}` first and the table/field after, the + wave-18 unconfigured-automation-column precedent exactly; `configured: false` rides the + wire so the surface says "finish setting this up" instead of snapping back to Manual. It + cannot fire while incomplete (the hooks match on the table it does not name), which is the + fail-closed direction. MALFORMED parts (an unknown comparison, a valueless compare, a + condition on a field the trigger does not watch) are still refused with the sentence — + incomplete is a state, wrong is not. + """ + if raw in (None, "", {}): + return (dict(previous) if isinstance(previous, dict) and previous else None), None + if not isinstance(raw, dict): + return None, "the trigger must be an object" + prev = previous if isinstance(previous, dict) else {} + key = _s(raw.get("key") or raw.get("kind") or prev.get("key") or prev.get("kind"), + 30).strip() + if key in TRIGGER_PLANNED: + # Declared on the wire, refused at the door — see the TRIGGER_PLANNED note. The sentence + # says WHY rather than "unknown trigger", because the picker legitimately showed it. + return None, (f"{TRIGGER_LABELS[key]!r} is on the list but not built yet. " + f"it renders so you can see it is coming, and it cannot be saved") + if key not in TRIGGER_KEYS: + return None, (f"{key or 'that trigger'!r} is not one of: " + ", ".join(TRIGGER_KEYS)) + if key == "schedule": + # ⛔ STILL NOT STORED, and the original reasoning holds for THIS key alone: `schedule` + # already owns the cron (`defn['schedule']` = `{cron, enabled}`), so a stored + # `{key:'schedule'}` would be a second copy of that fact, free to disagree with it. + return None, None + if key == "manual": + # ⭐⭐ 2026-08-07 (owner ruling) — **MANUAL IS A REAL, STORED CHOICE NOW.** + # Owner: *"Make it so that when you choose Manual, it IS a manual automation that the user + # can just press Run to make the full flow work."* + # + # ⛔ THIS SPLITS A PAIR THAT SHOULD NEVER HAVE BEEN ONE. The old line refused both keys + # together with one argument — *"storing a no-op trigger would be a second copy of that + # fact"* — and that argument is TRUE OF `schedule` AND FALSE OF `manual`. A schedule has + # another home; **manual has none.** Nothing anywhere recorded "this automation is + # manual", so storing it is not a duplicate: it is the only record there has ever been. + # + # ⚠ WHAT THE CONFLATION COST, measured live: picking Manual wrote nothing, so + # `chosen` (`!!trigger || schedule.enabled`) stayed false, the Builder kept showing the + # "nobody has decided yet" empty state, and Configuration — including the Database picker + # a plain automation cannot do without — never rendered. The owner reported it twice. The + # previous note reasoned that a manual option *"would bounce straight back to this state + # on the next reload"* and concluded the option should be HIDDEN; the honest conclusion + # was that it should be STORED. + # + # ⚠ DELIBERATELY BARE. No `enabled`, no `paused`: a manual trigger cannot be switched off + # (Run now always works, which is the whole of what it means) and a switch that governs + # nothing is worse than no switch. `graph()` keeps this node on the SCHEDULE panel so the + # cron stays reachable — picking Manual says how it fires today, never that it may not be + # scheduled tomorrow. + return {"key": "manual"}, None + out = {"key": key, + "enabled": bool(raw["enabled"]) if "enabled" in raw else + bool(prev.get("enabled", True)), + "paused": bool(raw["paused"]) if "paused" in raw else bool(prev.get("paused"))} + if key in TRIGGER_TABLE_KEYS: + table = _s(raw.get("table") if "table" in raw else prev.get("table"), 60).strip() + if table and not table.startswith(UT_PREFIX): + return None, ("event triggers watch blank databases (ut_*) this wave. " + f"{table!r} is not one") + out["table"] = table + if key == "event_field": + # ⭐ WAVE 24 · C-TRIG LAW 4 (owner item 7) — THE WATCHED FIELD IS GONE. "When a record + # matches conditions" is a CONDITION trigger and nothing else: the field picker made it a + # second, quieter way to express the same narrowing, and the owner asked for one. + # ⚠ MIGRATION, NEVER A REFUSAL (law 6). A stored `field` is simply not read, so it is + # dropped on this definition's next clean — silently, and exactly once, because nothing + # writes the key back. A refusal here would have 400'd the live automations that carry it. + cond, cerr = clean_cond(raw.get("when") if "when" in raw else prev.get("when"), + where="the trigger") + if cerr: + return None, cerr + out["when"] = cond + if key == "record_updated": + # Airtable's shape: watch named fields, or leave the list empty for "any field". Empty + # is the WIDER reading and it is the default there too, so it stays the default here. + watch_raw = raw.get("fields") if "fields" in raw else prev.get("fields") + if watch_raw in (None, ""): + watch = [] + elif not isinstance(watch_raw, list): + return None, "the watched-field list must be a list of field keys" + else: + watch = [_s(f, 80).strip() for f in watch_raw if _s(f, 80).strip()] + if len(watch) > MAX_WATCH_FIELDS: + return None, (f"a record-updated trigger watches at most {MAX_WATCH_FIELDS} " + f"fields. Leave the list empty to watch every field") + out["fields"] = watch + # ⭐ WAVE 24 · C-TRIG LAW 5 (owner item 7) — THE CONDITION IS GONE, and this REMOVES A + # SHIPPED CAPABILITY. Watched `fields` is now the whole of this trigger's configuration: + # "a record was updated" is an event, and asking it to also be a filter was the overlap + # with `event_field` the owner asked to end. Stated loudly in the contract AND here so + # nobody restores it as a bug fix. + # ⚠ Same migration shape as law 4: a stored `when` stops being read, so `_row_gate` + # naturally returns "no gate" for it — the write itself becomes the event — rather than + # this needing a second removal anywhere. + if key == "enters_view": + out["viewId"] = _s(raw.get("viewId") if "viewId" in raw else prev.get("viewId"), + 80).strip() + if key == "form_submitted": + # Blank = any form on that database. Naming one narrows to it, which is what a table + # carrying an intake form AND a correction form needs. + out["formToken"] = _s(raw.get("formToken") if "formToken" in raw + else prev.get("formToken"), 64).strip() + if key == "webhook": + # The token is MINTED here, once, and survives every later patch — rotating it on + # every Save would silently break the external caller the URL was given to. + out["token"] = _s(prev.get("token"), 64) or _secrets_token() + # ⭐ WAVE 24 — DEBT D-41: the request BODY, mapped onto record fields by config. + # ⚠ BOTH HALVES ARE OPTIONAL, and that is what keeps this additive: a webhook with no + # map behaves exactly as it did — it fires the flow and reads nothing — so the live + # webhook automations are untouched. `webhook` deliberately stays OUT of + # `TRIGGER_TABLE_KEYS`: joining it would make a table REQUIRED for `configured`, and + # every existing webhook trigger would go unconfigured and stop firing. + table = _s(raw.get("table") if "table" in raw else prev.get("table"), 60).strip() + if table and not table.startswith(UT_PREFIX): + return None, ("a webhook writes into a blank database (ut_*). " + f"{table!r} is not one") + out["table"] = table + fmap, ferr = clean_body_map(raw.get("fieldMap") if "fieldMap" in raw + else prev.get("fieldMap")) + if ferr: + return None, ferr + out["fieldMap"] = fmap + if key == "email": + out["query"] = _s(raw.get("query") if "query" in raw else prev.get("query"), + 200).strip() or "in:inbox is:unread" + out["configured"] = _trigger_configured(out) + return out, None + + +#: D-41 ceilings. 40 mapped cells is `MAX_ACTION_VALUES` doubled — a webhook payload is somebody +#: else's schema and is legitimately wider than an action's hand-written value list. +MAX_BODY_FIELDS = 40 +MAX_BODY_DEPTH = 5 + + +def clean_body_map(raw): + """D-41: `{"": ""}` for a webhook. Returns `(map, error)`. + + Paths are DOTTED into nested objects (`customer.email`). ⛔ NO ARRAY INDEXING in v1, stated + rather than half-supported: `items.0.sku` would read as working for the first element and + silently write nothing the day a payload arrives with the list empty, which is the shape of + bug this module keeps paying for. A path that resolves to nothing writes nothing. + """ + if raw in (None, ""): + return {}, None + if not isinstance(raw, dict): + return None, "the webhook field map must be an object of {body path: field key}" + if len(raw) > MAX_BODY_FIELDS: + return None, f"a webhook maps at most {MAX_BODY_FIELDS} values onto a record" + out = {} + for path, field in raw.items(): + p = _s(path, 200).strip() + if not p: + return None, "a webhook mapping has an empty body path" + if len(p.split(".")) > MAX_BODY_DEPTH: + return None, (f"{p!r} reaches more than {MAX_BODY_DEPTH} levels into the payload. " + f"map a shallower value") + fk = re.sub(r"[^a-z0-9_]+", "_", _s(field, 60).strip().lower()).strip("_") + if not fk: + return None, f"the value at {p!r} is not mapped to a field" + out[p] = fk[:60] + return out, None + + +def body_value(body, path): + """One dotted path into a decoded JSON body, or None. Scalars only — a mapped value that is + an object or a list answers None rather than a stringified `{...}` in a cell, because the + Row contract is scalar and a serialised dict in a grid cell is unreadable and unfilterable.""" + cur = body + for part in str(path or "").split("."): + if not isinstance(cur, dict): + return None + cur = cur.get(part) + return cur if isinstance(cur, (str, int, float, bool)) else None + + +def webhook_row(rt, defn, body): + """D-41: write ONE record from a webhook payload. Returns `(row_id, mapped_count, note)`. + + ⚠ `note` EXISTS BECAUSE THE CAP WAS SILENT. A table at its row ceiling returned the same + `("", 0)` as "no map configured" and the door answered a cheerful 200 — indistinguishable, + from the only side the caller is on, from a payload whose paths did not resolve. That is the + D-11 class (a table that quietly stops growing), and the caller here is a machine that will + keep posting. The note rides the 200: the flow still fires, and the answer says why no row + was written. + + ⛔ THROUGH THE ENGINE'S OWN WRITER, so it emits no row events — the structural loop + prevention law (A2(2)). A sibling automation watching this table does NOT fire on a + webhook-written row, exactly as it does not fire on a scrape's rows. The webhook's OWN flow + fires, because `hook_fire` fires it explicitly, which is the difference between "this + trigger fired" and "a write happened". + """ + trg = (defn or {}).get("trigger") or {} + table, fmap = str(trg.get("table") or ""), dict(trg.get("fieldMap") or {}) + if not table or not fmap or not isinstance(body, dict): + return "", 0, "" # no map configured — nothing to report + t = ut_get(rt, table) + if t is None: + return "", 0, f"{table} no longer exists, so nothing was written" + values = {} + for path, fkey in fmap.items(): + v = body_value(body, path) + if v is not None: + values[fkey] = _s(v, 500) if isinstance(v, str) else v + if not values: + return "", 0, ("none of the mapped paths resolved to a value in this payload. " + "check the paths against what you are sending") + rows = dict((t.get("rows") or {})) + if len(rows) >= row_cap(table): + return "", 0, (f"{table} is at its {row_cap(table)}-row limit, so no record was " + f"written (the flow still ran)") + rid = str(max([int(r) for r in rows if str(r).isdigit()] or [0]) + 1) + rows[rid] = values + ut_write_rows(rt, table, rows) + return rid, len(values), "" + + +def _trigger_configured(trg, config=None): + """Is this trigger complete enough to fire? One reader, because "configured" is asserted in + three places (the wire, the graph node, the hooks) and three copies of a boolean is how a + surface says "ready" about a trigger the engine skips. + + ⭐ WAVE 24 (A2) — `config` is OPTIONAL and only `ig_profile_match` reads it, because that is + the one trigger whose configuration lives in the DEFINITION's config (the discovery filters) + rather than on the trigger. `clean_trigger` calls this without it and so answers + conservatively (False); `clean_definition` calls it again with the validated config and + refines. Conservative-then-refined is the fail-closed order — the reverse would flash + "ready" on a trigger with nothing to search for. + """ + key = str((trg or {}).get("key") or "") + # ⭐ WAVE 29 — BOTH discovery triggers, and they answer identically: a corpus search with no + # filter is not a search, it is a request for the whole index. Named as a pair rather than + # `or`-ed onto the Instagram line so a third network joins by adding a key, not by editing a + # boolean expression. + if key in ("ig_profile_match", "tiktok_profile_match"): + return bool((config or {}).get("predicates")) + if key in TRIGGER_TABLE_KEYS and not trg.get("table"): + return False + if key == "event_field": + # C-TRIG law 4: the CONDITION is now the whole of it. No condition = "fire on anything, + # ever" — which is not a trigger, it is a description of the table. + # ⚠ STATED CONSEQUENCE OF THE MIGRATION: a live automation that narrowed by FIELD alone + # and carried no condition becomes `configured: false` on its next clean. It stops + # firing, and it SAYS SO — the graph node reads "Finish setting this trigger up before it + # can fire" and `configured` rides the wire. Visibly unfinished, never silently inert. + return bool(trg.get("when")) + if key == "enters_view": + return bool(trg.get("viewId")) + return True + + +def _secrets_token(): + import secrets as _sec + return _sec.token_urlsafe(24) + + +# --- the circuit breaker (A2(2)) — process memory, like _RUNNING: a counter that outlives the +# process would keep punishing an automation for a storm that ended with the restart. +_FIRES = {} +_FIRES_LOCK = threading.Lock() + + +def _breaker_trips(tenant, auto_id, now=None): + now = now if now is not None else time.time() + key = (tenant, str(auto_id)) + with _FIRES_LOCK: + log = [t for t in _FIRES.get(key, []) if now - t < FIRE_WINDOW_SECONDS] + log.append(now) + _FIRES[key] = log + return len(log) > FIRE_LIMIT + + +def _pause_trigger(rt, auto_id, note): + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + d = cur.get(str(auto_id)) + if d is not None: + trg = d.get("trigger") + if isinstance(trg, dict): + trg["paused"] = True + d["statusNote"] = _s(note, 200) + return cur + _store_update(rt, _up, flush="sync") + + +def trigger_fire(rt, tenant, auto_id, log=print, rows=None): + """One trigger firing — breaker first, then the ordinary async run. False when it did not + start (breaker, or already running — both are answers, not errors).""" + if _breaker_trips(tenant, auto_id): + note = (f"auto-paused: more than {FIRE_LIMIT} trigger fires in " + f"{FIRE_WINDOW_SECONDS // 60} minutes. Something is writing this trigger's " + f"subject in a loop") + _pause_trigger(rt, auto_id, note) + log(f"[aios-auto] breaker: {auto_id} {note}") + return False + return run_async(rt, tenant, auto_id, username="automation", log=log, rows=rows) + + +# --- the settle buffer (A2(1)) — per (tenant, automation), coalescing a burst into ONE +# evaluation. With EVENT_SETTLE_SECONDS == 0 the evaluation is INLINE (deterministic for gates). +_SETTLE = {} +_SETTLE_LOCK = threading.Lock() + + +def _settle_buffer(rt, tenant, auto_id, row_id, field="", after=None, log=print): + """Buffer ONE touched row for a coalesced evaluation, carrying the written cell. + + ⚠ WAVE 23 — THE WRITTEN VALUE IS STILL LOAD-BEARING, and an earlier draft of this wave + dropped it on the theory that `_settle_eval` could just read the row. It cannot, and the + reason is worth stating because it is invisible from this file: an ordinary ut cell typed at + the grid door lands in the editor's PER-USER OVERLAY stratum + (`grid_events.overlay_patch` → `table_store.patch_overlay`), not in the `user_tables` + definition rows. Only stage-field writes go through `patch_cells` to the shared rows. So for + the common case the value that just changed exists ONLY in this event, and a definition-row + read sees the pre-write value — the trigger would evaluate stale and never fire. + `_settle_eval` therefore MERGES: the definition row underneath (which is what lets a C4 tree + read the record's other columns) with the written cells on top. + """ + key = (tenant, str(auto_id)) + cell = {str(field): after} if field else {} + if EVENT_SETTLE_SECONDS <= 0: + with _SETTLE_LOCK: + buf = _SETTLE.setdefault(key, {"rows": {}}) + buf["rows"].setdefault(str(row_id), {}).update(cell) + _settle_eval(rt, tenant, auto_id, log=log) + return + with _SETTLE_LOCK: + buf = _SETTLE.setdefault(key, {"rows": {}}) + buf["rows"].setdefault(str(row_id), {}).update(cell) + timer = buf.get("timer") + if timer is not None: + timer.cancel() # the burst continues — push the window out + timer = threading.Timer(EVENT_SETTLE_SECONDS, _settle_eval, + args=(rt, tenant, auto_id), kwargs={"log": log}) + timer.daemon = True + buf["timer"] = timer + timer.start() + + +def view_filter(rt, table_key, view_id): + """`(tree, fields, problem)` for one saved view on a user table — the substrate the + `enters_view` trigger tests membership against (C3-v2 / owner R2). + + Personal strata first (`find_view`), then the shared bucket, because a view somebody shared + is exactly the kind an automation gets pointed at. A view that has been deleted answers a + PROBLEM rather than an empty tree: an empty tree matches everything, so degrading to one + would turn "when a record enters Overdue" into "on every write", which is the widening this + module refuses everywhere else. + """ + key = str(table_key or "") + vid = str(view_id or "").strip() + if not key or not vid: + return None, [], "the trigger names no view" + try: + import core.table_store as table_store + tops = table_store.make(f"{key}_table_workspace", st=rt) + found = tops.find_view(vid) + view = (found[1] if found else None) or tops.shared_view(vid) + except Exception as e: # noqa: BLE001 + return None, [], f"the view could not be read ({type(e).__name__})" + if not isinstance(view, dict): + return None, [], f"view {vid!r} no longer exists on {key}" + cfg = view.get("config") or {} + tree = {"nodes": cfg.get("filters") or [], "conj": cfg.get("filterConj") or "and"} + return tree, list((ut_get(rt, key) or {}).get("fields") or []), "" + + +def _row_gate(rt, defn, trg): + """The MATCH gate for a row-event trigger: `(row -> bool) | None`, plus a problem string. + + None means the trigger has NO match gate — the write itself is the event (wave 22's "the + field changed", and `record_updated` over any field). A problem means the gate cannot be + built, and the caller must then fire NOTHING: a gate we cannot evaluate is not a gate that + passes. + """ + key = trg.get("key") + if key == "enters_view": + tree, fields, problem = view_filter(rt, trg.get("table"), trg.get("viewId")) + if problem: + return None, problem + import harness.filter_eval as filter_eval + return (lambda row: filter_eval.matches(tree, row, fields)), "" + when = trg.get("when") + if when: + return (lambda row: lane_match(when, row)), "" + return None, "" + + +def _settle_eval(rt, tenant, auto_id, log=print): + """Evaluate one settled burst: edge over per-record armed state, flood hold, then fire.""" + with _SETTLE_LOCK: + buf = _SETTLE.pop((tenant, str(auto_id)), None) + written = dict((buf or {}).get("rows") or {}) + touched = list(written) + if not touched: + return + d = all_definitions(rt).get(str(auto_id)) + trg = (d or {}).get("trigger") or {} + if trg.get("key") not in ("event_field", "record_updated", "enters_view") \ + or trg.get("paused") or not trg.get("enabled", True) \ + or not trg.get("configured", True): + return + gate, problem = _row_gate(rt, d, trg) + if problem: + # LOUD, once, and it does not fire. A trigger pointed at a deleted view is a broken + # automation, not a quiet no-op — the note is what the surface shows instead of "On". + if (d.get("statusNote") or "") != problem: + _pause_trigger(rt, auto_id, problem) + log(f"[aios-auto] trigger gate: {auto_id} {problem}") + return + fired = [] + if gate is not None: + rows = (ut_get(rt, trg.get("table") or "") or {}).get("rows") or {} + disarmed = set((d.get("state") or {}).get("eventDisarmed") or []) + nxt = set(disarmed) + for rid in touched: + # The merge (see `_settle_buffer`): the shared definition row underneath so a C4 + # tree can read the record's other columns, the just-written cells on top because + # for an ordinary ut column this event is the ONLY place that value exists yet. + if gate({**(rows.get(rid) or rows.get(str(rid)) or {}), + **(written.get(rid) or {})}): + if rid not in disarmed: + fired.append(rid) # false→true THIS evaluation: the edge + nxt.add(rid) + else: + nxt.discard(rid) # left the state — re-armed (Airtable's rule) + if nxt != disarmed: + set_state(rt, auto_id, {"eventDisarmed": sorted(nxt)[:5000]}) + if not fired: + return + else: + fired = list(touched) # "the field changed" — the burst is the edge + if len(fired) > FLOOD_LIMIT: + _commit_run(rt, auto_id, "partial", + f"the trigger matched {len(fired)} records in one evaluation. More than " + f"the {FLOOD_LIMIT}-record flood hold, so nothing ran. Press Run now to " + f"process them deliberately", {"held": len(fired)}, True) + return + trigger_fire(rt, tenant, auto_id, log=log, rows=fired) + + +def _seed_event_state(rt, defn): + """A2(1)'s enable rule: records ALREADY matching when the trigger is set start DISARMED, so + turning the trigger on fires nothing — the first fire needs a real false→true transition. + Evaluated over the definition rows (the machine-written truth this trigger class watches). + + Wave 23: one seeder for all three gated triggers, reading the SAME `_row_gate` the evaluation + reads. Two implementations of "does this row match" is how a seed disagrees with the edge it + is supposed to arm, and the symptom would be a flood of fires the moment somebody enables it. + """ + trg = (defn or {}).get("trigger") or {} + if trg.get("key") not in ("event_field", "record_updated", "enters_view") \ + or not trg.get("configured"): + return + gate, problem = _row_gate(rt, defn, trg) + if gate is None or problem: + return # no match gate ⇒ nothing to arm; a broken gate seeds nothing + rows = (ut_get(rt, trg.get("table") or "") or {}).get("rows") or {} + matching = sorted(str(rid) for rid, row in rows.items() if gate(row or {})) + set_state(rt, defn.get("id"), {"eventDisarmed": matching[:5000]}) + + +def grid_hook(evt): + """THE listener the human write doors emit into (registered onto + `core.user_tables.ROW_HOOKS` by `routes_automation` at import — the one place that may + import both sides). Never raises into a write path; a broken trigger listener must not + break typing into a cell.""" + try: + st = evt.get("st") + if st is None: + return + tenant = str(getattr(st, "key", "") or "royal-imports") + table = str(evt.get("table") or "") + kind = str(evt.get("type") or "") + # ⭐⭐ WAVE 31 · T35 (D-134) — `cached=True`, and this is the ONLY caller that passes it. + # This function runs once per ROW EVENT, so a 20,000-row import used to perform 20,000 + # whole-document deep copies of the automations bucket, under the store lock, to re-read a + # trigger set that had not changed. See `all_definitions` for why the memo is safe here + # and nowhere else. + defs = all_definitions(st, cached=True) + field = str(evt.get("field") or "") + rid = str(evt.get("rowId") or "") + for aid, d in defs.items(): + trg = (d or {}).get("trigger") or {} + key = str(trg.get("key") or "") + if trg.get("paused") or not trg.get("enabled", True) \ + or not trg.get("configured", True) \ + or key not in TRIGGER_ROW_KEYS \ + or str(trg.get("table") or "") != table: + continue + if kind == "record_created" and key == "record_created": + hw = _ig_int((d.get("state") or {}).get("rcHighwater")) or 0 + ridn = int(rid) if rid.isdigit() else None + if ridn is None or ridn <= hw: + continue # once per record EVER (A2(4)) — undo-proof + set_state(st, aid, {"rcHighwater": ridn}) + # ⛔⛔ W31-T35 — MIRROR THE WRITE INTO THE MEMO'S OWN COPY, IN THE SAME STATEMENT. + # This is the hazard a definitions memo creates and the reason D-134 is not a + # one-line change: this branch READS `rcHighwater` and WRITES it, so within one + # import burst the second row would compare against the highwater the FIRST row + # set — and read the pre-write value out of the memo, fire again, and break + # A2(4)'s *"once per record EVER — undo-proof"*. `set_state` goes through + # `_store_update`, which drops the memo, but `defs` is the object already in hand + # for the rest of THIS event; keeping the two in step is what makes the memo safe + # rather than merely fast [[read-path-cannot-witness-write-path]]. + (d.setdefault("state", {}))["rcHighwater"] = ridn + trigger_fire(st, tenant, aid, rows=[rid]) + elif kind == "event_field" and key == "event_field": + # ⭐ WAVE 24 · law 4 — the per-FIELD narrowing is gone with the stored key. Every + # write on the watched table settles, and the CONDITION decides whether it fires + # (`_row_gate`). The old `not trg.get("field") or str(...) == field` test would + # now always take its first arm anyway; leaving a read of a key the validator no + # longer writes is the drift seat this migration exists to close. + _settle_buffer(st, tenant, aid, rid, field, evt.get("after")) + elif kind == "event_field" and key == "record_updated" and ( + not (trg.get("fields") or []) or field in (trg.get("fields") or [])): + _settle_buffer(st, tenant, aid, rid, field, evt.get("after")) + elif key == "enters_view": + # BOTH kinds feed it: a row can enter a view by being edited into its filter or + # by being CREATED already inside it. Listening only to edits would silently miss + # every new record — the half of the definition a reader assumes is covered. + _settle_buffer(st, tenant, aid, rid, field, evt.get("after")) + except Exception as e: # noqa: BLE001 + print(f"[aios-auto] trigger hook failed: {type(e).__name__}: {e}") + + +def form_fired(rt, table_key, row_id, values=None, form_token=""): + """⭐ THE FROZEN SIGNATURE session D calls from the public form door (contract C9/W23-W7). + + One submitted form row → every `form_submitted` automation watching that database fires. + Returns the list of automation ids that started, so the door can log what it set off (and so + the gate can assert it, rather than asserting a side effect nobody can see). + + ⛔ THIS IS A HUMAN DOOR, deliberately: an anonymous submission is a person filling in a form, + so it fires triggers exactly like typing into a cell does. The loop-prevention law is not + weakened by that — the engine's own writers still never reach here (only `routes_forms` calls + it), so an automation cannot create a form row and re-fire itself. + + `values` is accepted and unused today: the row is already written when this is called, and + the flow reads it from the table. It stays in the signature because the caller HAS it and a + later refire policy ("only when field X was submitted") needs it — a parameter added later + would mean changing D's call site in a wave that does not own it. + """ + started, table = [], str(table_key or "") + if not table: + return started + tenant = str(getattr(rt, "key", "") or "royal-imports") + for aid, d in all_definitions(rt).items(): + trg = (d or {}).get("trigger") or {} + if trg.get("key") != "form_submitted" or trg.get("paused") \ + or not trg.get("enabled", True) or not trg.get("configured", True): + continue + if str(trg.get("table") or "") != table: + continue + want = str(trg.get("formToken") or "") + if want and not hmac.compare_digest(want, str(form_token or "")): + continue # this automation watches a DIFFERENT form on that table + if trigger_fire(rt, tenant, aid, rows=[str(row_id)] if row_id else None): + started.append(aid) + return started + + +def hook_fire(rt, tenant, auto_id, token, body=None): + """The webhook trigger's decision, separated from FastAPI so the gate can drive it. + Returns `(status, payload)` — 404 unknown, 403 wrong/missing token or wrong trigger kind, + 409 already running, 200 started. + + ⭐ WAVE 24 (D-41): `body` is the decoded JSON payload, or None when the caller sent none or + sent something that is not JSON. It is written onto a record BEFORE the flow fires — the + flow's actions walk the record the webhook just created, which is the whole point of mapping + it. `body=None` is the pre-wave behaviour exactly, so an existing caller is unaffected. + """ + import hmac as _hmac + defn = all_definitions(rt).get(str(auto_id)) + if defn is None: + return 404, {"error": "unknown_automation"} + trg = defn.get("trigger") or {} + want = str(trg.get("token") or "") + if trg.get("key") != "webhook" or not want: + return 403, {"error": "no_webhook", "message": + "this automation has no webhook trigger"} + if trg.get("paused") or not trg.get("enabled", True): + return 403, {"error": "webhook_off", "message": "the webhook trigger is turned off"} + if not _hmac.compare_digest(want, str(token or "")): + return 403, {"error": "bad_token", "message": "that token is not valid"} + # D-41: map the payload onto a record FIRST, so the flow that fires next walks it. + row_id, mapped, note = webhook_row(rt, defn, body) + started = trigger_fire(rt, tenant, auto_id, rows=[row_id] if row_id else None) + out = {"started": bool(started), "at": _iso(), + # Answered even when zero, so a caller wiring a map up can see whether their paths + # resolved. Silence here would make "my JSON is not landing" undebuggable from the + # outside, which is the only side the caller is on. + "rowId": row_id or None, "mapped": mapped} + if note: + out["note"] = note # a 200 that wrote no row SAYS which reason it was + return 200, out + + +def email_poll(rt, tenant, auto_id, defn, log=print, _list=None, _read=None): + """The email trigger's tick half (C3): poll the CREATOR's Gmail through C5's seam, write a + row per NEW matching message, fire the flow. `_list`/`_read` are injection points so the + gate drives this without a network; production leaves them None. + + Fail-closed and QUIET when unconnected: the statusNote says so ONCE (not a run entry per + tick — 96 identical failures a day is a klaxon, not a status). Bounded everywhere: at most + `EMAIL_MAX_PER_POLL` new messages per tick, the flood hold above that, one coalesced write. + """ + trg = defn.get("trigger") or {} + if trg.get("key") != "email" or trg.get("paused") or not trg.get("enabled", True): + return None + import oauth_connect + creator = str(defn.get("createdBy") or "").strip() or "admin" + token, err = oauth_connect.google_creds(rt, creator) + if err: + if (defn.get("statusNote") or "") != err: + def _note(cur): + cur = cur if isinstance(cur, dict) else {} + dd = cur.get(str(auto_id)) + if dd is not None: + dd["statusNote"] = _s(err, 200) + return cur + _store_update(rt, _note, flush="sync") + return None + lister = _list or (lambda q, n: oauth_connect.gmail_list(token, q, n)) + reader = _read or (lambda mid: oauth_connect.gmail_message(token, mid)) + ids, lerr = lister(trg.get("query") or "", EMAIL_MAX_PER_POLL + FLOOD_LIMIT) + if lerr: + return _commit_run(rt, auto_id, "partial", f"the Gmail poll did not answer. {lerr}", + {}, True) + seen = set((defn.get("state") or {}).get("emailSeen") or []) + fresh = [m for m in ids if m not in seen] + if not fresh: + return None # nothing new is not a run — no history spam + if len(fresh) > FLOOD_LIMIT: + return _commit_run(rt, auto_id, "partial", + f"{len(fresh)} new emails matched in one poll. More than the " + f"{FLOOD_LIMIT}-record flood hold, so nothing was written. Narrow " + f"the query, or press Run now after adjusting it", + {"held": len(fresh)}, True) + fresh = fresh[:EMAIL_MAX_PER_POLL] + rows_in, notes = [], [] + for mid in fresh: + row, rerr = reader(mid) + if row: + rows_in.append(row) + elif rerr: + notes.append(rerr) + table_key = (defn.get("config") or {}).get("targetTable") or "" + if not table_key: + return _commit_run(rt, auto_id, "error", + "the email trigger has nowhere to write. The automation names no " + "target database", {}, False) + ut_ensure(rt, (defn.get("config") or {}).get("targetLabel") or defn.get("name") or "Inbox", + EMAIL_FIELDS, username=str(defn.get("createdBy") or "automation"), key=table_key) + existing = dict((ut_get(rt, table_key) or {}).get("rows") or {}) + merged, counts = upsert_rows(existing, rows_in, "email_id", cap=row_cap(table_key)) + ut_write_rows(rt, table_key, merged) + new_seen = (list(seen) + fresh)[-EMAIL_SEEN_CAP:] + set_state(rt, auto_id, {"emailSeen": new_seen}) + summary = (f"{len(fresh)} new email{'' if len(fresh) == 1 else 's'} matched. " + f"{counts['inserted']} row{'' if counts['inserted'] == 1 else 's'} written") + if notes: + summary += f". {notes[0][:100]}" + entry = _commit_run(rt, auto_id, "ok" if not notes else "partial", summary, counts, + True, affected=list(merged)[:200]) + trigger_fire(rt, tenant, auto_id, log=log) + return entry + + +def compose_sentence(defn): + """The one-sentence server-composed summary (airtable-brief rec 7): rendered from the + definition so it cannot lie about what runs.""" + cfg = defn.get("config") or {} + trg = defn.get("trigger") or {} + sched = defn.get("schedule") or {} + kind = defn.get("kind") + if trg.get("key") == "event_field": + # Law 4: no watched field any more, so the sentence stops naming one. It said + # "When None changes on ut_x" the moment the key stopped being stored. + head = f"When a record in {trg.get('table')} matches conditions" + elif trg.get("key") == "record_updated": + head = f"When a record in {trg.get('table')} is updated" + elif trg.get("key") == "ig_profile_match": + head = "When an Instagram profile fits the criteria" + elif trg.get("key") == "tiktok_profile_match": + head = "When a TikTok profile fits the criteria" + elif trg.get("key") == "record_created": + head = f"When a record is created in {trg.get('table')}" + elif trg.get("key") == "webhook": + head = "When the webhook is called" + elif trg.get("key") == "email": + head = f"When an email matches {trg.get('query')}" + elif sched.get("enabled"): + head = f"{_cron_label(sched.get('cron'))}" + else: + head = "When you press Run now" + if kind == "scrape_db": + host = urlparse(str(cfg.get("url") or "")).hostname or "the page" + body = f"read {host} and upsert rows into {cfg.get('targetTable') or 'a new database'}" + elif kind == "field_instagram": + # ⚠ NO RUNG CLAUSE (R5). There is one way to capture a profile now, so ", exact counts + # first" / ", anonymous only" described a choice that no longer exists. + body = f"capture Instagram profiles for {cfg.get('targetTable') or 'the database'}" + elif kind in DISCOVERY_KINDS: + # ⭐⭐ WAVE 30 · T05 — the arm covers both kinds, and the network is NAMED from the kind + # rather than hard-coded into the sentence. Instagram-only, a TikTok search fell into the + # `plain` branch below and introduced itself as *"do nothing yet — this automation has no + # actions"*: a flatly false sentence, on the one surface whose stated promise is that it + # cannot lie about what runs. (The `plain` branch's own comment records the mirror-image + # incident — it USED to be this arm, and described every plain automation as an Instagram + # search for 0 profiles. The same two branches have now mis-described each other's + # automations in both directions, which is why neither may be a fallthrough.) + _dplatform, _dtable, _, _ = discovery_facts(kind) + body = (f"search {_dplatform} for up to {cfg.get('recordsLimit') or 0} profiles into " + f"{cfg.get('targetTable') or _dtable}") + else: + # ⭐ WAVE 24 — `plain` describes itself from its FLOW, because the flow is all it has. + # ⛔ THIS BRANCH USED TO BE `discover_instagram`'s, so before the arm above existed every + # plain automation would have introduced itself as "search Instagram for up to 0 profiles + # into ut_ig_candidates" — a sentence composed from a definition that says none of it, + # on the one surface whose whole promise is that it "cannot lie about what runs". + n = sum(1 for _ in walk_actions((defn.get("flow") or {}).get("actions"))) + tbl = cfg.get("targetTable") or trg.get("table") or "" + body = ((f"run {n} action{'' if n == 1 else 's'}" + (f" on {tbl}" if tbl else "")) + if n else "do nothing yet. This automation has no actions") + lanes = cfg.get("lanes") or [] + tail = f", then route each record across {len(lanes)} lanes" if lanes else "" + return f"{head}, {body}{tail}." + + +def run_now(rt, tenant, auto_id, username="automation", log=print, rows=None): + """Execute one automation SYNCHRONOUSLY. The route wraps this in a thread; the tick calls it + directly. Returns the run entry, or None when it was already running (the 409).""" + defn = all_definitions(rt).get(str(auto_id)) + if defn is None: + return None + # ⛔⛔ WAVE 32 · T45 (owner item 10) — AN UNCONFIGURED ACTION BLOCKS THE RUN, HERE, WHERE EVERY + # DOOR PASSES. The route checks too so a person gets a 400 rather than a silent no-op, but the + # tick and the webhook do not go through the route; a client-only block is not a block (D-112). + # ⚠ BEFORE `_claim`, deliberately: claiming and then refusing would leave the automation + # marked running until the release, i.e. a refusal that also produces a phantom 409 for the + # next honest attempt. + _refusal = run_refusal(defn) + if _refusal: + log(f"[aios-auto] refused: {_refusal}") + return None + if not _claim(tenant, auto_id): + return None + # ⛔ THE TABLES AN AUTOMATION MAKES BELONG TO THE AUTOMATION'S CREATOR, not to whoever + # happened to press Run — and above all not to the scheduler, which is not a person and + # cannot own anything (see `ut_ensure`). Without this the owner of a database was decided by + # whether a human or a cron got to the first run first. + owner = str(defn.get("createdBy") or "").strip() + if owner and username in MACHINE_OWNERS: + username = owner + try: + _step(tenant, auto_id, "running") + # A deferred metric snapshot is paid work already in Bright Data's queue. Collect it + # first and do not start another profile scrape while it is outstanding: re-running the + # action would buy duplicate engagement reads and reintroduce the timeout this handoff + # exists to remove. The normal scheduler calls this path too via `pending_collect_ids`. + # ⭐ 2026-08-09 — THE PROFILE HANDOFF IS COLLECTED FIRST, for the same reason and one + # rung earlier: a profile snapshot the vendor is still building is paid work, and + # starting a fresh scrape for the same handle would buy the identical row a second time. + # Ahead of the metric collector because the profile IS the thing the run was asked for; + # the engagement batches hang off it. + if _pending_profile_tasks(defn): + state, summary, counts, affected, steps = collect_pending_profile_snapshots( + rt, defn, username=username, log=log, + step=lambda text: _step(tenant, auto_id, text)) + return _commit_run(rt, auto_id, state, summary, + {k: v for k, v in counts.items() if k != RUN_NOTES_KEY}, + state != "error", affected, steps, + notes=counts.get(RUN_NOTES_KEY)) + if _pending_metric_tasks(defn): + state, summary, counts, affected, steps = collect_pending_metric_snapshots( + rt, defn, username=username, log=log, + step=lambda text: _step(tenant, auto_id, text)) + return _commit_run(rt, auto_id, state, summary, + {k: v for k, v in counts.items() if k != RUN_NOTES_KEY}, + state != "error", affected, steps, + notes=counts.get(RUN_NOTES_KEY)) + runner = RUNNERS.get(defn.get("kind")) + if runner is None: + return _commit_run(rt, auto_id, "error", + f"unknown automation kind {defn.get('kind')!r}", {}, False) + try: + # ⭐ WAVE 24 (item 6, on D's measurement) — THE LIVE STEP, closed over this run. + # `status.step` was already on the wire and D's half renders it; measured against the + # code, it was set exactly ONCE ("running") and never again, so the word would have + # been identical whether a run was mid-vendor-wait or genuinely hung. Rendering a + # constant as a progress indicator is worse than rendering nothing: it looks like an + # answer. The runners move it now, and the 120 s Bright Data wait counts out loud. + state, summary, counts, affected, steps = runner( + rt, defn, username=username, log=log, + step=lambda text: _step(tenant, auto_id, text), rows=rows) + except Refused as e: + return _commit_run(rt, auto_id, "error", f"refused: {e}", {}, False) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] run {auto_id} failed: {type(e).__name__}: {e}") + return _commit_run(rt, auto_id, "error", + f"{type(e).__name__}: {str(e)[:200]}", {}, False) + # ⭐ WAVE 23 (C4/C5) — THE FLOW RUNS HERE, after the machine steps and before the run is + # committed, over the records this run actually touched. ONE call site rather than three + # inside the runners: every kind gets actions and endings for free, and a fourth runner + # cannot forget to opt in. + # + # ⚠ Its absence was the wave's most expensive near-miss: actions were stored, validated, + # wired to the wire and covered by twelve gate checks that all called `apply_actions` + # DIRECTLY — so the whole feature was green and unreachable. A person would have built a + # flow, pressed Run now, and watched nothing happen. The gate now drives `run_now`. + # + # A failing action must not fail the RUN: the machine steps already wrote their rows and + # reporting that as an error would misdescribe what happened. It degrades to `partial` + # with the reason in the summary — the cap_note discipline. + # ⚠ BOUND BEFORE THE `try`. The `except` below falls through to the same `_commit_run`, + # which now reads this name — an assignment only on the success path would turn any + # action failure into a NameError inside the handler that exists to prevent exactly that. + # ⚠ THE RUNNER PRODUCES NOTES TOO, and its are the ones that survive a run which walked + # NOTHING — the branch where every candidate was a known-dead handle, i.e. exactly the + # run a person stares at wondering why the automation stopped doing anything. + run_notes = list((counts or {}).pop(RUN_NOTES_KEY, None) or []) + try: + a_counts = apply_actions(rt, defn, _flow_table(defn), affected or [], + username=username, log=log, + step=lambda text: _step(tenant, auto_id, text)) + # ⭐⭐ D-103 — POPPED BEFORE THE MERGE. The per-record reasons ride inside `counts` so + # the runner contract keeps its shape, and they must leave before the merge or they + # would be a "count" everywhere downstream. + run_notes += list(a_counts.pop(RUN_NOTES_KEY, None) or []) + counts = {**(counts or {}), **{k: v for k, v in a_counts.items() if v}} + if a_counts.get("enrichMetricBatchesPending"): + batches = int(a_counts["enrichMetricBatchesPending"]) + state = "partial" if state != "error" else state + summary += (f". {batches} post-engagement batch" + f"{'' if batches == 1 else 'es'} still building; Views and other " + "metrics will be collected automatically without another paid read") + if a_counts.get("enrichUnbound"): + # ⛔ D-79(2): AND THE FIX GOES IN THE SUMMARY, not only in the log. The run is + # `partial` because it genuinely did part of its job — it walked the records — and + # the sentence names the ONE thing that has to change, in the two places a person + # can change it. The old behaviour was `ok` with an empty table. + state = "partial" if state != "error" else state + summary += (". The Instagram step did not run: this database has no profile " + "column. Name one on the step, or mark a text column as the " + "Instagram profile" + + _unbound_hint(rt, _flow_table(defn))) + if a_counts.get("ttEnrichUnbound"): + # ⛔ WAVE 30 · T08 — ITS OWN SENTENCE, not the one above with a word swapped by a + # variable. A flow may carry BOTH steps, and the fix a person has to apply is + # per-column: naming an Instagram profile column does nothing for a TikTok step, + # so a single sentence covering "the enrich step" would send them to the wrong + # place half the time. Both may appear on one run, which is correct. + state = "partial" if state != "error" else state + summary += (". The TikTok step did not run: this database has no TikTok profile " + "column. Name one on the step, or mark a text column as a TikTok " + "profile" + + _unbound_hint(rt, _flow_table(defn))) + if a_counts.get("ttEnrichBlocked"): + state = "partial" if state != "error" else state + tt_note = next((n for n in run_notes if "(TikTok): " in n), "") + summary += (f". {int(a_counts['ttEnrichBlocked'])} TikTok profile read(s) were " + "blocked" + (f". {_s(tt_note, 220)}" if tt_note else "")) + if a_counts.get("enrichProfileBatchesPending"): + # ⭐ The paid profile the vendor is still building. Said out loud so a run that + # looks like a failure is read as the handoff it is — the tick finishes it. + n = int(a_counts["enrichProfileBatchesPending"]) + state = "partial" if state != "error" else state + summary += (f". {n} profile read{'' if n == 1 else 's'} took longer than the " + "wait allows and will be collected automatically, at no extra cost") + if a_counts.get("enrichBlocked"): + state = "partial" if state != "error" else state + # ⭐⭐ D-103 — THE REASON IS IN THE SENTENCE, not only behind a click. "1 profile + # read(s) were blocked" is the exact string the owner read three mornings running + # before asking "wtf is going on"; it names a quantity and withholds the one + # thing that would let anybody act. The first note is the vendor's own words. + # ⚠ ...and the Instagram selector SKIPS the tagged TikTok lines for the same + # reason. Two sentences quoting each other's vendor reason is worse than one. + blocked_note = next((n for n in run_notes + if ": " in n and "(TikTok): " not in n), "") + summary += (f". {int(a_counts['enrichBlocked'])} profile read(s) were blocked" + + (f". {_s(blocked_note, 220)}" if blocked_note else "")) + # ⭐ WAVE 25 · C5 — A FULL TARGET IS A `partial` RUN THAT SAYS SO. D-11 made this the + # law for the runners' OWN writes (`cap_note`), and `create_record` never joined: it + # logged the cap and rolled up `ok`, so a flow that had silently stopped writing + # looked exactly like one that had nothing to write. Same rule, same sentence shape. + if a_counts.get("createCapped"): + state = "partial" if state != "error" else state + summary = (f"{summary}. {a_counts['createCapped']} row(s) NOT created: a target " + f"database is at its row cap") + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] actions on {auto_id} failed: {type(e).__name__}: {e}") + state = "partial" if state != "error" else state + summary = f"{summary}. The actions did not finish ({type(e).__name__})" + return _commit_run(rt, auto_id, state, summary, counts, state != "error", affected, + steps, notes=run_notes) + finally: + _release(tenant, auto_id) + + +def run_async(rt, tenant, auto_id, username="automation", log=print, rows=None): + """Start a run on a background thread. False when one is already in flight (→ 409).""" + if running(tenant, auto_id): + return False + th = threading.Thread(target=run_now, args=(rt, tenant, auto_id, username, log, rows), + daemon=True, name=f"automation-{tenant}-{auto_id}") + th.start() + return True + + +# --------------------------------------------------------------------------------------------- +# THE TICK + the in-process scheduler +# --------------------------------------------------------------------------------------------- + +def pending_collect_ids(rt): + """⭐ 2026-08-06 — automations holding a snapshot the vendor is still building. + + ⛔ THE DEFECT THIS CLOSES, reported live: *"why is the SMALL 10 records test search taking + forever, its not populating"*. A corpus search takes ~20 minutes, so the run hands off and + stores `pendingSnapshot` with the summary *"The next run picks up the results"* — which is + true only if there IS a next run. The owner's automation is MANUAL (`schedule.enabled` false), + so `due_ids` never returns it, nothing ever collected it, and the results the account had + already been charged for sat ready at the vendor forever. The sentence was not wrong; it was + describing a run nobody had scheduled. + + A pending snapshot is unfinished work the tenant has already paid for, so the tick finishes it + regardless of schedule. Independent of `due_ids` on purpose — a schedule says *start something + new*, this says *collect what is already running*, and folding the second into the first would + make an unscheduled automation's paid result depend on someone remembering to press a button. + """ + out = [] + for aid, d in all_definitions(rt).items(): + if not isinstance(d, dict): + continue + if (d.get("trigger") or {}).get("paused"): + continue + # ⭐⭐ WAVE 30 · T05 — BOTH discovery kinds, and this one is a MONEY defect rather than a + # cosmetic one. It tested `== "discover_instagram"`, so a TikTok corpus search stored its + # `pendingSnapshot`, told the person *"The next run picks up the results"*, and was then + # never returned by this function — the tick collected nothing, forever. That is EXACTLY + # the live incident quoted in the docstring above (*"why is the SMALL 10 records test + # search taking forever, its not populating"*), reproduced for the second platform by the + # wave that added it: a result the tenant has already been charged for, stranded. + # ⚠ NOT ON THE SCOUT'S LIST OF FIVE. Found by reading this function for a different + # ticket, which is the argument for `DISCOVERY_KINDS` in one line — the sites that test a + # kind string are not enumerable by memory, and this one is three thousand lines from the + # others. Pressing Run again does still collect (the runner's own branch reads the same + # field), so the money was recoverable BY HAND and only ever silently lost on a schedule. + pending_discovery = (d.get("kind") in DISCOVERY_KINDS + and str((d.get("state") or {}).get("pendingSnapshot") or "").strip()) + # ⭐ 2026-08-09 — PROFILE handoffs join the other two. Without this line the profile + # deferral would be stored and never collected, which is the same defect it fixes wearing + # a queue: `due_ids` only returns SCHEDULED automations, and the automation this was + # measured on is `trigger: manual`. A capability written down but never walked is what + # this whole change is about, so it must not be reintroduced one function later. + if pending_discovery or _pending_metric_tasks(d) or _pending_profile_tasks(d): + out.append(aid) + return sorted(out) + + +def due_ids(rt, now=None): + """Which of this tenant's automations a tick at `now` should start. Pure over the store.""" + return sorted(aid for aid, d in all_definitions(rt).items() if is_due(d, now)) + + +def tick(rt, tenant, now=None, log=print): + """Fire every due automation for ONE tenant + poll every email trigger (C3). Returns the + ids started. The email polls are bounded and fail-quiet per automation — one broken + mailbox connection must not stop the tenant's schedules.""" + started = [] + # ⭐ COLLECT-FIRST (2026-08-06). A snapshot the vendor has finished building is a result the + # tenant has already been charged for; it is collected whether or not this automation is on a + # schedule. `_claim` makes the union safe — an id in both lists starts once. + for aid in dict.fromkeys(list(pending_collect_ids(rt)) + list(due_ids(rt, now))): + if run_async(rt, tenant, aid, username="scheduler", log=log): + started.append(aid) + for aid, d in all_definitions(rt).items(): + if (d.get("trigger") or {}).get("key") == "email": + try: + if email_poll(rt, tenant, aid, d, log=log) is not None: + started.append(aid) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] email poll {tenant}/{aid} failed: {type(e).__name__}: {e}") + # C7's amendment: date-window metrics advance with the tick, so `today` is never staler + # than one tick while a scheduler exists. A tenant without metric fields pays a dict scan. + try: + refresh_metrics(rt, log=log) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] metric refresh {tenant} failed: {type(e).__name__}: {e}") + # ⭐ 2026-08-07 — the relational pass, on the same tick and for the same reason. A rollup on + # table A goes stale when table B gains a row, and A cannot know that happened; the tick is + # the only place that sees both. ⚠ SEPARATE try/except from the metrics above deliberately — + # one pass failing must not silently cancel the other, which a shared block would do. + try: + refresh_relations(rt, log=log) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] relation refresh {tenant} failed: {type(e).__name__}: {e}") + # ⭐ THE READ-THROUGH ROLLUPS, on the same tick and for a stronger version of the same reason. + # A linked rollup goes stale when the LINKED TABLE gains a row; a source-backed one goes stale + # when ODOO does — and its window moves on its own besides (a `ytd` column is wrong on 1 + # January without anybody writing anything). The tick is the only place that sees either. + # + # ⛔ ITS OWN try/except, like the two passes above, and the reason is the same one stated + # there: this pass REFUSES loudly on a truncated group set, and a shared block would let that + # honest refusal silently cancel the relational pass that had already succeeded. + # + # ⚠ IT WRITES NOTHING WHEN IT REFUSES — see `rollup_sql`'s header. A half-applied rollup mixes + # two vintages of one column and looks completely normal, which is why the refusal is total. + try: + import rollup_sql + for _tk in list((rt.get(UT_STORE_KEY) or {})): + if rollup_sql.source_fields((rt.get(UT_STORE_KEY) or {}).get(_tk) or {}): + # ⚠ `today` is DEFAULTED INSIDE `compute`, not passed: this scope has no + # such local, and `refresh_metrics` above resolves it the same way. + n = rollup_sql.compute(rt, _tk) + if n: + log(f"[aios-auto] source rollups {tenant}/{_tk}: {n}") + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] source rollup {tenant} failed: {type(e).__name__}: {e}") + if started: + log(f"[aios-auto] tick {tenant}: started {', '.join(started)}") + return started + + +def tick_all(now=None, log=print): + """Every registered tenant. Fail-quiet per tenant: one tenant's broken store must not stop + the others' schedules.""" + from harness import runtime as _rt + out = {} + for slug in _rt.known_tenants(): + try: + out[slug] = tick(_rt.get_runtime(slug), slug, now=now, log=log) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] tick {slug} skipped: {type(e).__name__}: {e}") + return out + + +#: How often the in-process scheduler wakes. A minute is the cron resolution; anything finer +#: would be a busy loop against a vocabulary that cannot express it. +TICK_SECONDS = int(os.environ.get("AIOS_AUTOMATION_TICK_SECONDS") or 60) +_SCHEDULER = [None] + + +def scheduler_loop(log=print): + """The resync-daemon pattern (`api/main.py:313`): sleep FIRST, then work. + + Sleeping first is deliberate and load-bearing for the gate battery — `verify_api` and + `verify_seam` import this module through `main.py`, run against fake stores in seconds and + exit. A loop that ticked on entry would fire inside them. + """ + while True: + time.sleep(TICK_SECONDS) + try: + tick_all(log=log) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] scheduler tick failed: {type(e).__name__}: {e}") + + +def start_scheduler(log=print): + """Start the loop once per process. **OPT-IN: `AIOS_AUTOMATIONS=1`.** + + ⚠ AMENDED 2026-08-03 (D), and the reason is a measurement rather than a preference. The wave + brief specified this thread DEFAULT-ON for the API. Then `verify_api.py` was timed: **196 + seconds**, i.e. more than three 60 s tick intervals. A default-on scheduler therefore fires + two or three times inside a gate that imports `main.py` — and while a tick over a fake store + finds nothing due, `tick_all` reaches `runtime.get_runtime()`, which BUILDS tenants and + mutates the LRU cache that `verify_api`'s own isolation assertions read. A background thread + that can move a gate's subject is a flaky suite waiting to happen. + + So it takes the convention `main.py:353` already established for exactly this hazard — + `AIOS_PREWARM=1` — for exactly the reason stated there: env-gated rather than a startup + event, so importing this module in a gate can never fire anything. The DEPLOY sets it; the + external EventBridge tick (R5) does not depend on it either way, since that POSTs the + endpoint rather than riding this thread. + """ + if os.environ.get("AIOS_AUTOMATIONS") != "1": + return False + if _SCHEDULER[0] is not None: + return False + th = threading.Thread(target=scheduler_loop, kwargs={"log": log}, + daemon=True, name="automation-scheduler") + _SCHEDULER[0] = th + th.start() + return True diff --git a/api/connectors_tt.py b/api/connectors_tt.py index 60d96945a35f7fd8ca79443bf70cdec87e0a4c53..0603f4f23f7075252fdcef06ce19371ac1620523 100644 --- a/api/connectors_tt.py +++ b/api/connectors_tt.py @@ -1,568 +1,568 @@ -"""connectors_tt.py — the TIKTOK connector (wave 29 · item 7 · DEBT D-9 · rulings R1 + R2). - -Everything in this file knows what a VENDOR's TikTok row looks like. Nothing in it knows what an -automation is. That split is `connectors_ig.py`'s (wave 27 item 23) and it is the reason this file -exists at all rather than another thousand lines inside the engine. - -⛔ **EVERY VENDOR FIELD NAME HERE WAS PROBED, NOT GUESSED.** The whole schema — 40 profile / 43 -post / 17 comment fields, each with the vendor's own type, description and `pii` flag — was read -live from `GET /datasets/{id}/metadata` for **$0.00** and written down in -`.claude/wiki/waves/wave29/proto/tiktok-schema.md` (promoted to `tiktok-capture.md` at -close-out). That document is the AUTHORITY: do not re-probe it, and do not invent a key. Where a -name below reads through a candidate list it is because the vendor has two names for one fact -(`biography`/`signature`, `region`/`country`), never because the name is uncertain. - -⛔ **NOTHING HERE EVER AUTHENTICATES TO TIKTOK.** No login, no cookie, no account to get banned — -public data through a supplier, exactly the rail `connectors_ig.py` states for Instagram. The -vendor key is a key to a SUPPLIER. - -⭐ **THE TRANSPORT IS `connectors_bd.py` — SHARED, VENDOR-NAMED, AND NO LONGER BORROWED FROM THE -OTHER PLATFORM'S CONNECTOR** (WAVE 30 · T09, DEBT D-128). `bd_call`, `bd_scrape` and -`bd_filter_start` take the dataset id as a PARAMETER — they are Bright Data's wire, not -Instagram's — and re-implementing them here would be a second copy of the deferral handling, the -truncation guard, the SSRF rail and the snapshot-progress reader, i.e. five places for one bug. -Until wave 30 the code was right and the NAME was wrong: this file imported thirteen symbols from -`connectors_ig`, which read as a dependency on Instagram and was really a dependency on a supplier. -⚠ **This file now imports ZERO names from `connectors_ig`, and a gate check asserts that**, because -the sentence above is the kind that quietly stops being true. - -⚠ **WHAT $0 COULD NOT BUY, so nobody reads this file as more measured than it is:** - 1. the real ROW shape — `/metadata` describes a DATASET, and Instagram's rows carry undeclared - envelope keys (`timestamp`, `input`) that no metadata call mentions; - 2. that a declared field POPULATES — Bright Data's Instagram Reels *declares* `views: number` - and delivers an account-grain wrong number (§4e). **Declared is not delivered**, and the one - TikTok claim that matters most (`play_count`) is exactly a declaration. -""" -from __future__ import annotations - -import automation_engine as engine -# ⚠ T09 — FOUR NAMES CAME OFF THIS LIST AND NOTHING BROKE, which is the point of deriving an -# import block from the AST both ways. `bd_call`, `bd_filter_start`, `bd_key` and `bd_ready` were -# imported here under a comment claiming they were *"re-exported for the runners"*; no runner ever -# read them off this module (the engine imports them from the transport itself), so they were four -# lines of dependency nobody was paying for. `[[artifact-with-no-importer]]` in its smallest form. -from connectors_bd import ( - _bd_first_url, - _bd_flag, - _bd_list, - _bd_source_payload, - _first, - _ig_int, - bd_scrape, -) - -# --------------------------------------------------------------------------------------------- -# THE DATASETS -# --------------------------------------------------------------------------------------------- -# ⚠ CATALOGUE PRESENCE IS NOT ENTITLEMENT — the same finding Instagram produced. `GET -# /datasets/list` returned 1,735 rows of which 12 are TikTok; the three below answer 200 with a -# full field list, and the two DISCOVERY halves answer **404 for our key**: -# `gd_lj71gn6l68bz7y9hc` (posts by profile) and `gd_lilwhto81z415d9mdl` (posts by keyword). -# ⇒ TikTok discovery routes through the PROFILES dataset's corpus filter, exactly as Instagram's -# does. A ticket that reaches for a by-keyword endpoint is reaching for a 404. -TT_DS_PROFILES = "gd_l1villgoiiidt09ci" # TikTok - Profiles. 40 fields, 152,000,000 records -TT_DS_POSTS = "gd_lu702nij2f790tmv9h" # TikTok - Posts. 43 fields -TT_DS_COMMENTS = "gd_lkf2st302ap89utw5k" # TikTok - Comments. 17 fields - -#: The vendor's two post-type tokens, verbatim from the dataset's own `ai_description` -#: (*"strictly these two"*, video = 99.5% of rows). Ours are `image`/`video`/`carousel`. -TT_POST_TYPE_VIDEO = "video" -TT_POST_TYPE_CONTENT = "content" - -#: What a TikTok profile URL looks like, for the runner that has a handle and needs a URL. Kept -#: beside the dataset ids because it is the same class of vendor fact. -#: ⚠ ONE SPELLING. `platform/core/user_tables.profile_url(handle, 'tiktok')` builds the identical -#: string from `_PROFILE_RULES` (contract C2, E's half) — this exists for the connector's own -#: batch calls, and the gate asserts the two agree rather than trusting that they do. -TT_PROFILE_URL = "https://www.tiktok.com/@{handle}" - - -def tt_profile_url(handle): - """`nurilab` → `https://www.tiktok.com/@nurilab`. `''` for a blank handle, never a bare `@`.""" - h = str(handle or "").strip().lstrip("@") - return TT_PROFILE_URL.format(handle=h) if h else "" - - -# --------------------------------------------------------------------------------------------- -# THE FIELD MAPS -# --------------------------------------------------------------------------------------------- -# Each function turns ONE vendor row into the cell dict for one of the `ut_tt_*` schemas declared -# in `automation_engine`. The rules they all obey, stated once: -# -# * **BLANK MEANS NOT READ, NEVER "THEY HAVE NONE".** A key the vendor did not send is OMITTED, -# so a later, richer pull fills it instead of being overwritten by this one's silence. `_first` -# returns `None` (never 0) when nothing matches, which is what makes that possible. -# * **A zero from the vendor is a MEASUREMENT and survives.** (Instagram's `_ig_zero_is_blank` -# rule is scoped to a paid rung whose zeros were proven fictional; nothing here has earned it.) -# * **Every unpromoted vendor key stays whole in `source_payload`.** A schema addition on the -# vendor's side is preserved rather than silently discarded while our column model catches up. -# * **Nothing here writes a session token.** `tt_chain_token`, `secu_id` (~85% null), `short_id` -# (100% null in the sample), `ftc` (100% null) and `relation` are deliberately unmapped; they -# ride in `source_payload` where they make no claim. - - -def _tt_str(node, *names): - """The first non-empty string among `names`, or None when the vendor sent nothing. - - ⛔ `None`, NOT `""`. The callers below drop `None` keys, which is what keeps a blank honest — - an empty string written into a cell claims "we looked and it is empty". - """ - v = _first(node, *names) - if v is None: - return None - s = str(v).strip() - return s or None - - -def _tt_pct(node, *names): - """A vendor 0–1 engagement fraction → our stored 0–100 percentage, or None. - - ⛔ THE ×100 IS NOT COSMETIC (wave 26, amendment C1-a). Our `pct` renderer appends the sign to - the STORED number, so writing the vendor's raw 0.0656 would print a 6.6% creator as `0.0%` — - measured on the Instagram side, and TikTok sends the same shape on all three of its rates. - """ - v = _first(node, *names) - if v is None: - return None - out = engine._pct100(v) - return out or None - - -def _tt_day(node, *names): - """A vendor stamp → `YYYY-MM-DD`, or None. Our `date` columns store a day.""" - v = _first(node, *names) - if v is None: - return None - return engine._day(v) or None - - -def _drop_blanks(row): - """The one place a mapped row loses its `None`s — see the BLANK MEANS NOT READ rule above.""" - return {k: v for k, v in row.items() if v is not None and v != ""} - - -def normalize_profile(node, handle=""): - """A TikTok Profiles row → the `ut_tt_profile` / `ut_tt_snapshots` cell shape. - - ⚠ TWO FIELDS READ THROUGH A CANDIDATE PAIR, and both pairs are the vendor's, not a guess: - * `biography` is PRIMARY and `signature` the FALLBACK — the probe measured `signature` - populated on 85% of rows and they carry the same text; - * `region` is PRIMARY and `country` the FALLBACK — `region` is the one with a documented - two-letter-ISO description, `country` has no description at all. - ⚠ `videos_count` is mapped to `posts_count` with a caveat recorded rather than hidden: its - `ai_description` ranges 1-89, so it may be a WINDOW rather than a lifetime total. It is the - only count of its kind the dataset offers. - """ - node = node if isinstance(node, dict) else {} - account = _tt_str(node, "account_id") or str(handle or "").strip().lstrip("@") - return _drop_blanks({ - "platform": engine.PLATFORM_TIKTOK, - "handle": account, - "full_name": _tt_str(node, "nickname"), - "tt_id": _tt_str(node, "id"), - "profile_url": _tt_str(node, "url") or (tt_profile_url(account) or None), - "bio": _tt_str(node, "biography", "signature"), - "external_url": _bd_first_url(_first(node, "bio_link")), - "verified": _bd_flag(node, "is_verified"), - "is_private": _bd_flag(node, "is_private"), - # ⚠ APPROXIMATE, and the word is the vendor's: `is_commerce_user` has *"many null values"* - # on their own description. It is the closest thing TikTok has to Instagram's - # `is_business_account`, and `_bd_flag` writes nothing at all when the key is absent — so - # the approximation only ever fills a cell the vendor actually answered. - "is_business": _bd_flag(node, "is_commerce_user"), - "followers": _ig_int(_first(node, "followers")), - "following": _ig_int(_first(node, "following")), - "posts_count": _ig_int(_first(node, "videos_count")), - # ⚠ `likes` on a PROFILE row is likes RECEIVED across the account's videos (18-110,200, no - # nulls). It is not a post-level number and it is not our `likes` column, which is why it - # is stored under a different name. - "likes_received": _ig_int(_first(node, "likes")), - "avg_engagement": _tt_pct(node, "awg_engagement_rate"), - "like_engagement": _tt_pct(node, "like_engagement_rate"), - "comment_engagement": _tt_pct(node, "comment_engagement_rate"), - "country_code": _tt_str(node, "region", "country"), - "region": _tt_str(node, "region"), - "predicted_lang": _tt_str(node, "predicted_lang"), - # ⚠ ACCOUNT AGE, NOT A MEASUREMENT STAMP — `create_time` on a profile is when the ACCOUNT - # was made. TikTok stamps nothing with "when this number was true", exactly like Instagram, - # which is why the append law dates a snapshot by when WE read it. - "account_created_at": _tt_day(node, "create_time"), - "source_payload": _bd_source_payload(node), - }) - - -def tt_post_type(node): - """A TikTok post row → one of OUR three type options, or None. - - ⛔ THE FIRST OF THE PROBE DOC'S TWO NAMED BLOCKERS. The vendor's vocabulary is `"video"` / - `"content"`; ours is `image` / `video` / `carousel` and has no `"content"`. Writing the - vendor's token would fail `_clean_field`'s option check on the way in and would put an - untranslated API word in front of a user on the way out. - - So: `video` is `video`, and `content` — TikTok's photo-mode post — is `image`, EXCEPT when the - row carries more than one `carousel_images` entry, which is what a carousel IS on either - network. ⚠ The multi-image branch is decided from the IMAGES, never from the type token: the - token cannot express it, so inferring `carousel` from the word would be inventing a fact. - ⚠ An UNKNOWN token returns None rather than defaulting to `video` (99.5% of rows are video, and - that is exactly what would make the wrong default invisible). - """ - raw = str((node or {}).get("post_type") or "").strip().lower() - if raw == TT_POST_TYPE_VIDEO: - return "video" - if raw != TT_POST_TYPE_CONTENT: - return None - images = (node or {}).get("carousel_images") - return "carousel" if isinstance(images, list) and len(images) > 1 else "image" - - -def normalize_post(node): - """A TikTok Posts row → the `ut_tt_posts` cell shape. None when it carries no identity. - - ⛔ THE SECOND NAMED BLOCKER, RESOLVED HERE AND NOWHERE ELSE: `play_count` is ONE number and - Instagram's schema has TWO columns for it (`plays` and `views`). On TikTok they are the same - fact — `play_count` IS the count TikTok displays under a video — so it maps to `views` and - `ut_tt_posts` HAS NO `plays` COLUMN. Copying one vendor number into two of our columns would - manufacture a second measurement that a rollup could average or double-count, which is a worse - outcome than the missing column it would paper over. - - ⚠ `shortcode` reads `shortcode` then `post_id`: both are 19-digit numerics on this dataset and - the probe measured them as the same shape. That equality is what lets the comments→posts link - join with NO normaliser, which the Instagram side never had. - ⚠ `num_share_count` (a number) is preferred over `share_count` (typed TEXT by the vendor). - ⚠ `commerce_info` is a business/commerce LOCATION per its own description — cities and - countries. It is NOT a paid-partnership flag, and nothing in this dataset is: TikTok declares - no equivalent, so `paid_partnership`/`partner` have no column on this family at all. - """ - node = node if isinstance(node, dict) else {} - shortcode = _tt_str(node, "shortcode", "post_id") - if not shortcode: - return None - return _drop_blanks({ - "platform": engine.PLATFORM_TIKTOK, - "shortcode": shortcode, - # ⛔⛔ `account_id`, NOT `profile_username` — MEASURED on a real Posts row 2026-08-12. - # The vendor's `profile_username` is the DISPLAY NAME (`"Dina"`), while `account_id` is the - # @handle (`"d1na_th"`) — the same field `normalize_profile` already reads for `handle`, so - # one name means one thing across both corpora. Reading the display name silently broke the - # only join this table has: `ut_tt_posts.influencer_key` -> `ut_tt_profile.handle` matched - # NOTHING, so a person could not filter posts by creator and a rollup would count zero. - # ⚠ NO FALLBACK TO `profile_username`, deliberately. It is not a degraded handle, it is a - # different fact, and filling a join key with it is worse than leaving it blank — a blank - # is visibly missing, a display name looks like an answer [[one-question-two-normalizers]]. - # The URL is the honest second source: it carries the handle by construction. - "influencer_key": (_tt_str(node, "account_id") - or tt_handle(_tt_str(node, "profile_url", "url") or "") or None), - "posted_at": _tt_day(node, "create_time"), - "type": tt_post_type(node), - "caption": _tt_str(node, "description"), - "url": _tt_str(node, "url"), - "hashtags": _bd_list(node, "hashtags"), - "tagged_location": _tt_str(node, "commerce_info"), - "views": _ig_int(_first(node, "play_count")), - "likes": _ig_int(_first(node, "digg_count")), - "comments": _ig_int(_first(node, "comment_count")), - "shares": _ig_int(_first(node, "num_share_count")), - "saves": _ig_int(_first(node, "collect_count")), - "video_duration": _ig_int(_first(node, "video_duration")), - "source_payload": _bd_source_payload(node), - }) - - -def normalize_comment(node): - """A TikTok Comments row → the `ut_tt_comments` cell shape. None without a comment id. - - ⚠ THE NAME COLLISION, RESOLVED: TikTok's `replies` is an ARRAY of reply objects and OUR - `replies` column is an INT count. The count comes from `num_replies`; the array stays whole in - `source_payload`. Reading the array's length instead would be a second, disagreeing answer to - a question the vendor already answers — and it would disagree, because a page of replies is not - all of them. - ⚠ `date_created` is typed `date` by this vendor, unlike Instagram's `comment_date` which is - text and needs defensive parsing. It still goes through `_tt_day` — one date path, so a vendor - that changes its mind cannot change ours. - ⚠ The comment TEXT and every identifiable commenter field (`commenter_user_name` is flagged - PII) stay in `source_payload` and are promoted to no column, which is the same posture the - Instagram comment schema takes for the same D-24 reason. - """ - node = node if isinstance(node, dict) else {} - comment_key = _tt_str(node, "comment_id") - if not comment_key: - return None - return _drop_blanks({ - "platform": engine.PLATFORM_TIKTOK, - "comment_key": comment_key, - "shortcode": _tt_str(node, "post_id"), - # ⭐⭐ OWNER RULING 2026-08-12: the comment's CONTENT gets a column. `comment_text` is the - # vendor's own key and `comment_text_only` its stripped variant (the probe recorded both); - # primary first, so a row carrying the rich form is not silently served the plain one. - # ⛔ The commenter's identity is deliberately NOT promoted — see `TT_COMMENT_FIELDS`. - "text": _tt_str(node, "comment_text", "comment_text_only"), - "commented_at": _tt_day(node, "date_created"), - "likes": _ig_int(_first(node, "num_likes")), - "replies": _ig_int(_first(node, "num_replies")), - "source_payload": _bd_source_payload(node), - }) - - -#: ⭐ The three maps, addressable by name — so a gate (and the runners in T04-T06) can walk them -#: rather than naming three functions, and so adding a fourth dataset is one entry. -TT_NORMALIZERS = { - "tt_profile": normalize_profile, - "tt_post_metrics": normalize_post, - "tt_comments": normalize_comment, -} - - -# --------------------------------------------------------------------------------------------- -# THE FETCH — wave 30 · W30-T08 (carrying wave-29's dropped T05) -# --------------------------------------------------------------------------------------------- - -def tt_handle(url): - """A TikTok profile URL **or** a bare handle → the handle. `''` when it is neither. - - Deliberately permissive about the input and strict about the output, because the two callers - hand it different things: an automation stores whatever a person typed in the profile column - (`@nurilab`, `nurilab`, or the full URL), while the discovery runner already holds a clean - `account_id`. One normaliser, so a row found by discovery and a row typed by hand cannot - resolve to two different handles. - """ - s = str(url or "").strip() - if not s: - return "" - if "tiktok.com" in s.lower(): - # Everything after the first `@`, up to the next path segment or query. - tail = s.split("@", 1)[1] if "@" in s else "" - s = tail.split("/")[0].split("?")[0].split("#")[0] - s = s.strip().lstrip("@").strip() - # A handle is the vendor's `account_id` shape: alphanumerics, dots and underscores. - return s if s and all(c.isalnum() or c in "._" for c in s) else "" - - -def tt_post_urls(node, limit=0): - """⭐ WAVE 30 · T10 — the profile row's own post permalinks, newest-first as the vendor sends. - - ⛔ THIS IS WHY TIKTOK POST CAPTURE COSTS NO EXTRA DISCOVERY. `top_videos` rides the PROFILE row - we have already bought, so the posts read is a scrape of links we hold, never a search for them. - The two TikTok DISCOVERY datasets (posts-by-profile, posts-by-keyword) are **404 for our key**, - so a design that reached for either would not merely be dearer, it would not work. - - ⛔⛔ CORRECTED 2026-08-12 — THIS DOCSTRING USED TO SAY *"the probe MEASURED `top_videos` as an - array of video permalinks, NO empties"*, AND THAT SENTENCE IS WHAT SHIPPED THE BUG. The probe - read `/datasets/{id}/metadata` — a DATASET description — and the phrase quoted was the field's - `ai_description`, not an observation of a row. The real row sends **dicts keyed `video_url`** - (measured below), so the reader built against the quoted sentence found nothing, forever, in - silence. ⭐ The transferable half: *"the probe measured X"* and *"the probe read a declaration - of X"* are different claims, and prose cannot be told apart by a reader downstream — which is - why the correction names the method, not just the value. - - ⚠ `top_posts_data` is deliberately NOT read: the probe calls it *"a thin dup of `top_videos`"*, - and preferring whichever happened to be longer is how one creator's window silently differs - from another's. - ⚠ `limit <= 0` means "everything the row carried". The CAP IS THE CALLER'S — `config.maxPosts`, - validated 1..12 — and it is applied here rather than after the scrape so an unwanted post is - never bought. [[a-constant-two-features-share]]: the 12 is the vendor's measured profile window, - not a number this function may invent. - """ - raw = (node or {}).get("top_videos") - out = [] - for item in raw if isinstance(raw, list) else []: - # ⛔⛔ MEASURED ON A REAL ROW 2026-08-12, AND IT IS NOT WHAT THE SCHEMA SAID. - # `top_videos` is NOT an array of permalink strings. One paid TikTok Profiles scrape of a - # live handle returns **19 DICTS**, keyed - # `video_url · video_id · playcount · diggcount · commentcount · share_count · - # favorites_count · create_date · cover_image`. - # The docstring above cites the $0 probe as having "measured" permalinks — it had not, and - # could not: `/datasets/{id}/metadata` describes a DATASET, and the probe's own verdict says - # so in terms (*"what $0 cannot buy … the real ROW shape … that a declared field - # POPULATES"*, `wave29/proto/tiktok-schema.md`). This is the SECOND time this vendor's - # declaration has diverged from its delivery on this exact axis; BD's IG Reels `views` was - # the first. [[reachable-is-not-the-same-as-built]] - # ⇒ The consequence, live: `TT_DS_POSTS` was never reached, because this returned an EMPTY - # list on every real profile — post capture could not have worked for anybody, and the - # T10 gate stayed green because its canned fixture encoded the DECLARED shape. A fixture - # written from a schema tests the schema. - # ⚠ `video_url` is FIRST because it is the key the vendor actually sends; `url` is kept - # because it costs nothing and is what a future corpus revision would most likely use. The - # bare-string branch stays for the same reason — this widens what is ACCEPTED and invents - # nothing: a shape that yields no `http…` value still degrades to "no posts", exactly as - # before, rather than to a URL built out of a guess. - # ⚠ `top_posts_data` is STILL not read (it carries `post_url` and would work): preferring - # whichever array happened to be longer is how one creator's window silently differs from - # another's, and that reasoning is unchanged by this correction. - if isinstance(item, dict): - u = str(item.get("video_url") or item.get("url") or "").strip() - else: - u = str(item or "").strip() - if u.lower().startswith("http") and u not in out: - out.append(u) - return out[:limit] if limit and limit > 0 else out - - -def pull_posts_tt(post_urls, log=print, deferred=None): - """The TikTok Posts dataset for a list of permalinks → `(rows, note)`, already normalised. - - ⚠ ONE CALL FOR THE WHOLE WINDOW. `bd_scrape` has always taken a list, and the Instagram side - measured what happens when a caller forgets: 25 records, one billed snapshot each, a walk still - running at 67 minutes. Nothing here loops per URL. - """ - urls = [str(u) for u in (post_urls or []) if str(u or "").strip()] - if not urls: - return [], "" - rows, note = bd_scrape(TT_DS_POSTS, urls, deferred=deferred) - if note: - log(f"[aios-tt] posts: {note}") - return [], note - out = [r for r in (normalize_post(n) for n in rows) if r] - return out, "" - - -def pull_comments_tt(post_urls, log=print, deferred=None): - """The TikTok Comments dataset for a list of POST permalinks → `(rows, note)`, normalised. - - ⛔ THE MOST EXPENSIVE THING THIS PRODUCT BUYS, and the reason `commentMetrics` defaults OFF on - both networks: a comments scrape ingests identifiable third parties who never entered anybody's - list (D-24). The mapper already keeps every commenter field in `source_payload` and promotes - none of them to a column; this function adds no new exposure, it just has to be asked for. - """ - urls = [str(u) for u in (post_urls or []) if str(u or "").strip()] - if not urls: - return [], "" - rows, note = bd_scrape(TT_DS_COMMENTS, urls, deferred=deferred) - if note: - log(f"[aios-tt] comments: {note}") - return [], note - out = [r for r in (normalize_comment(n) for n in rows) if r] - return out, "" - - -#: ⭐⭐ WAVE 30 · D-156 — THE MEDIA DATASETS, AS A SET, SO THE HAND-OFF CAN FILTER ON IDENTITY. -#: `_media_deferrals` uses this to lift ONLY posts/comments snapshots out of the local deferral -#: list. That is what makes it structurally impossible to file a PROFILE snapshot in the engine's -#: metric queue — the defect a draft of T10 shipped and A-39 booked as "the wrong fix is worse -#: than the gap". A membership test cannot be got wrong by a later edit the way `if` order can. -TT_MEDIA_DATASETS = (TT_DS_POSTS, TT_DS_COMMENTS) - - -def _media_deferrals(deferred): - """The POSTS/COMMENTS entries of a `bd_scrape` deferral list — never the profile's. - - ⚠ The engine, not this module, decides what a deferral MEANS: it stamps `kind` and the - handle and files it. TikTok needs no `_tag_metric_deferrals` twin because one dataset is one - kind here, so the id already carries everything a mapper choice depends on — and importing - Instagram's tagger is not available anyway (W30-T09 gates ZERO `from connectors_ig` lines). - """ - out = [] - for d in deferred or []: - if isinstance(d, dict) and str(d.get("datasetId") or "") in TT_MEDIA_DATASETS: - out.append(dict(d)) - return out - - -def pull_profile_tt(url, log=print, pending_profile=None, prefetch=None, - max_posts=0, post_metrics=False, comment_metrics=False): - """ONE TikTok profile from the vendor. Same return contract as `pull_profile`. - - `{state, profile, posts, comments, via, note}` with `state ∈ ok | partial | blocked | error`, - so the engine's enrich branch treats every network identically and no caller learns a new - shape. - - ⭐ WAVE 30 · T10 — POSTS AND COMMENTS ARE REAL NOW, AND BOTH DEFAULT OFF, exactly as Instagram's - do. `post_metrics` scrapes the profile row's own `top_videos` permalinks (see `tt_post_urls` — - no discovery call, because both TikTok discovery datasets 404 for our key); `comment_metrics` - then scrapes the comments of the posts that came back. ⚠ COMMENTS REQUIRE POSTS by construction - rather than by a rule: their input IS a post permalink, so asking for comments with post capture - off is a request with no subject, and it returns none instead of quietly buying posts nobody - asked for. - - ⛔ `partial` IS THE SUCCESS STATE WHENEVER NO MEDIA WAS READ, and that is deliberate rather than - pessimistic. The Instagram contract reads `ok` only when identity AND media both landed - (`pull_profile_bd`: *"identity without media is still partial ... a run that wrote a follower - count and no posts must not paint green over a posts table that did not grow"*). So: posts not - ASKED for → `partial`, saying so; posts asked for and landed → `ok`; asked for and none came → - `partial` with the vendor's reason. The state answers "did this pull deliver what it went for", - never "did the function finish". - - ⚠ **NO FREE RUNG, AND NO FALLBACK CHAIN.** Instagram's `pull_profile` drops to Apify when the - paid rung refuses; `providers.DEFAULT_CHAINS["tt_profile"]` is deliberately single-provider, - with its own note explaining that a multi-provider chain is a promise something walks it and - that nothing walks Instagram's second name today either. So a refusal here is final, and it - says so instead of implying a retry somewhere. - """ - handle = tt_handle(url) - if not handle: - return {"state": "error", "profile": {}, "posts": [], "comments": [], "via": "", - "note": f"{url!r} is not a TikTok profile URL or handle"} - - # ⭐ THE BATCH FAST PATH, same shape as the Instagram side: `prefetch` is `{handle: node}` from - # one multi-URL scrape covering a whole selection. A hit is a vendor round trip that does not - # happen; a miss falls through to the single-URL call below. - cached = prefetch.get(handle) if isinstance(prefetch, dict) else None - _deferred = [] - if isinstance(cached, dict) and cached: - rows, note = [cached], "" - else: - rows, note = bd_scrape(TT_DS_PROFILES, [tt_profile_url(handle)], deferred=_deferred) - - node = rows[0] if rows else {} - profile = normalize_profile(node, handle) if node else {} - # ⛔ THE READABILITY TEST IS `followers`/`following`, NOT "did we get a dict". `normalize_profile` - # drops blanks, so an unreadable row still returns `{"platform": …, "handle": …}` — truthy, and - # carrying nothing anybody asked for. The Instagram rung tests exactly this pair for exactly - # this reason, and answering "0 followers" instead is the failure it exists to prevent. - unreadable = profile.get("followers") is None and profile.get("following") is None - if note or unreadable: - # ⭐ THE DEFERRAL IS HANDED OVER RATHER THAN DISCARDED. A snapshot the vendor is still - # building HAS ALREADY BEEN PAID FOR; dropping its id bills again on the next run for the - # same record. That was live on the Instagram profile path until 2026-08-09 — measured on - # nurilab as two runs, two fresh snapshots, both abandoned — and it is not being - # reintroduced here by omission. - if isinstance(pending_profile, list): - for d in _deferred: - pending_profile.append({**d, "kind": "profile", "influencer": handle}) - why = note or ("the scrape answered, but no follower/following counts were readable in it " - "(the field names may have moved - see tiktok-capture.md)") - return {"state": "blocked", "profile": {}, "posts": [], "comments": [], "via": "brightdata", - "deferredProfile": [d.get("snapshotId") for d in _deferred], - "note": why} - - # --- W30-T10: THE MEDIA, ONLY WHEN IT WAS ASKED FOR. ------------------------------------ - if not post_metrics: - return {"state": "partial", "profile": profile, "posts": [], "comments": [], - "via": "brightdata", - "note": note or "profile read; post capture is off for this step"} - urls = tt_post_urls(node, limit=max_posts) - if not urls: - # ⚠ NOT AN ERROR AND NOT A RETRY. A creator with no `top_videos` has nothing to buy, and - # saying so is what stops the next run paying to be told the same thing. - return {"state": "partial", "profile": profile, "posts": [], "comments": [], - "via": "brightdata", - "note": note or "profile read; this account's row carried no post links"} - posts, p_note = pull_posts_tt(urls, log=log, deferred=_deferred) - comments, c_note = ([], "") - if comment_metrics and posts: - # The comments dataset is keyed on a POST permalink, so it reads the posts we just bought — - # `url` from the mapper, never the profile's raw array, so a post the posts scrape refused - # is not silently asked about again one rung later. - comments, c_note = pull_comments_tt([p.get("url") for p in posts if p.get("url")], - log=log, deferred=_deferred) - # ⭐⭐ WAVE 30 · D-156 — THE MEDIA DEFERRALS ARE HANDED BACK, and the shape of the hand-off is - # the whole lesson. An earlier draft of T10 appended every `_deferred` entry to - # `pending_profile` tagged `kind: "profile"`. By the time control reaches here a PROFILE - # deferral is impossible — the profile branch above returns `blocked` on any note — so **every - # id fanned out that way was a POSTS or COMMENTS snapshot in the PROFILE queue**, whose - # collector writes preset profile cells onto somebody's record from post rows. The engine keeps - # the two queues apart deliberately (`_pending_profile_tasks` vs `_pending_metric_tasks`). - # ⇒ So this returns them under their OWN key, filtered by dataset identity - # (`_media_deferrals`), and the engine files them in the metric queue with the handle it - # already holds. Returning rather than appending also keeps the queue's vocabulary out of a - # connector: this module knows which CORPUS deferred, never what the engine calls it. - # ⚠ `deferredMedia` rides BOTH returns on purpose. The empty-posts case is the one that - # matters most — that is exactly the run where the vendor took too long, so a caller reading - # the ids only from the success path would lose every batch it actually paid for. - deferred_media = _media_deferrals(_deferred) - if not posts: - return {"state": "partial", "profile": profile, "posts": [], "comments": [], - "via": "brightdata", "deferredMedia": deferred_media, - "note": p_note or note or "profile read; the post source returned nothing"} - return {"state": "ok", "profile": profile, "posts": posts, "comments": comments, - "via": "brightdata", "deferredMedia": deferred_media, - "note": c_note or note or ""} +"""connectors_tt.py — the TIKTOK connector (wave 29 · item 7 · DEBT D-9 · rulings R1 + R2). + +Everything in this file knows what a VENDOR's TikTok row looks like. Nothing in it knows what an +automation is. That split is `connectors_ig.py`'s (wave 27 item 23) and it is the reason this file +exists at all rather than another thousand lines inside the engine. + +⛔ **EVERY VENDOR FIELD NAME HERE WAS PROBED, NOT GUESSED.** The whole schema — 40 profile / 43 +post / 17 comment fields, each with the vendor's own type, description and `pii` flag — was read +live from `GET /datasets/{id}/metadata` for **$0.00** and written down in +`.claude/wiki/waves/wave29/proto/tiktok-schema.md` (promoted to `tiktok-capture.md` at +close-out). That document is the AUTHORITY: do not re-probe it, and do not invent a key. Where a +name below reads through a candidate list it is because the vendor has two names for one fact +(`biography`/`signature`, `region`/`country`), never because the name is uncertain. + +⛔ **NOTHING HERE EVER AUTHENTICATES TO TIKTOK.** No login, no cookie, no account to get banned — +public data through a supplier, exactly the rail `connectors_ig.py` states for Instagram. The +vendor key is a key to a SUPPLIER. + +⭐ **THE TRANSPORT IS `connectors_bd.py` — SHARED, VENDOR-NAMED, AND NO LONGER BORROWED FROM THE +OTHER PLATFORM'S CONNECTOR** (WAVE 30 · T09, DEBT D-128). `bd_call`, `bd_scrape` and +`bd_filter_start` take the dataset id as a PARAMETER — they are Bright Data's wire, not +Instagram's — and re-implementing them here would be a second copy of the deferral handling, the +truncation guard, the SSRF rail and the snapshot-progress reader, i.e. five places for one bug. +Until wave 30 the code was right and the NAME was wrong: this file imported thirteen symbols from +`connectors_ig`, which read as a dependency on Instagram and was really a dependency on a supplier. +⚠ **This file now imports ZERO names from `connectors_ig`, and a gate check asserts that**, because +the sentence above is the kind that quietly stops being true. + +⚠ **WHAT $0 COULD NOT BUY, so nobody reads this file as more measured than it is:** + 1. the real ROW shape — `/metadata` describes a DATASET, and Instagram's rows carry undeclared + envelope keys (`timestamp`, `input`) that no metadata call mentions; + 2. that a declared field POPULATES — Bright Data's Instagram Reels *declares* `views: number` + and delivers an account-grain wrong number (§4e). **Declared is not delivered**, and the one + TikTok claim that matters most (`play_count`) is exactly a declaration. +""" +from __future__ import annotations + +import automation_engine as engine +# ⚠ T09 — FOUR NAMES CAME OFF THIS LIST AND NOTHING BROKE, which is the point of deriving an +# import block from the AST both ways. `bd_call`, `bd_filter_start`, `bd_key` and `bd_ready` were +# imported here under a comment claiming they were *"re-exported for the runners"*; no runner ever +# read them off this module (the engine imports them from the transport itself), so they were four +# lines of dependency nobody was paying for. `[[artifact-with-no-importer]]` in its smallest form. +from connectors_bd import ( + _bd_first_url, + _bd_flag, + _bd_list, + _bd_source_payload, + _first, + _ig_int, + bd_scrape, +) + +# --------------------------------------------------------------------------------------------- +# THE DATASETS +# --------------------------------------------------------------------------------------------- +# ⚠ CATALOGUE PRESENCE IS NOT ENTITLEMENT — the same finding Instagram produced. `GET +# /datasets/list` returned 1,735 rows of which 12 are TikTok; the three below answer 200 with a +# full field list, and the two DISCOVERY halves answer **404 for our key**: +# `gd_lj71gn6l68bz7y9hc` (posts by profile) and `gd_lilwhto81z415d9mdl` (posts by keyword). +# ⇒ TikTok discovery routes through the PROFILES dataset's corpus filter, exactly as Instagram's +# does. A ticket that reaches for a by-keyword endpoint is reaching for a 404. +TT_DS_PROFILES = "gd_l1villgoiiidt09ci" # TikTok - Profiles. 40 fields, 152,000,000 records +TT_DS_POSTS = "gd_lu702nij2f790tmv9h" # TikTok - Posts. 43 fields +TT_DS_COMMENTS = "gd_lkf2st302ap89utw5k" # TikTok - Comments. 17 fields + +#: The vendor's two post-type tokens, verbatim from the dataset's own `ai_description` +#: (*"strictly these two"*, video = 99.5% of rows). Ours are `image`/`video`/`carousel`. +TT_POST_TYPE_VIDEO = "video" +TT_POST_TYPE_CONTENT = "content" + +#: What a TikTok profile URL looks like, for the runner that has a handle and needs a URL. Kept +#: beside the dataset ids because it is the same class of vendor fact. +#: ⚠ ONE SPELLING. `platform/core/user_tables.profile_url(handle, 'tiktok')` builds the identical +#: string from `_PROFILE_RULES` (contract C2, E's half) — this exists for the connector's own +#: batch calls, and the gate asserts the two agree rather than trusting that they do. +TT_PROFILE_URL = "https://www.tiktok.com/@{handle}" + + +def tt_profile_url(handle): + """`nurilab` → `https://www.tiktok.com/@nurilab`. `''` for a blank handle, never a bare `@`.""" + h = str(handle or "").strip().lstrip("@") + return TT_PROFILE_URL.format(handle=h) if h else "" + + +# --------------------------------------------------------------------------------------------- +# THE FIELD MAPS +# --------------------------------------------------------------------------------------------- +# Each function turns ONE vendor row into the cell dict for one of the `ut_tt_*` schemas declared +# in `automation_engine`. The rules they all obey, stated once: +# +# * **BLANK MEANS NOT READ, NEVER "THEY HAVE NONE".** A key the vendor did not send is OMITTED, +# so a later, richer pull fills it instead of being overwritten by this one's silence. `_first` +# returns `None` (never 0) when nothing matches, which is what makes that possible. +# * **A zero from the vendor is a MEASUREMENT and survives.** (Instagram's `_ig_zero_is_blank` +# rule is scoped to a paid rung whose zeros were proven fictional; nothing here has earned it.) +# * **Every unpromoted vendor key stays whole in `source_payload`.** A schema addition on the +# vendor's side is preserved rather than silently discarded while our column model catches up. +# * **Nothing here writes a session token.** `tt_chain_token`, `secu_id` (~85% null), `short_id` +# (100% null in the sample), `ftc` (100% null) and `relation` are deliberately unmapped; they +# ride in `source_payload` where they make no claim. + + +def _tt_str(node, *names): + """The first non-empty string among `names`, or None when the vendor sent nothing. + + ⛔ `None`, NOT `""`. The callers below drop `None` keys, which is what keeps a blank honest — + an empty string written into a cell claims "we looked and it is empty". + """ + v = _first(node, *names) + if v is None: + return None + s = str(v).strip() + return s or None + + +def _tt_pct(node, *names): + """A vendor 0–1 engagement fraction → our stored 0–100 percentage, or None. + + ⛔ THE ×100 IS NOT COSMETIC (wave 26, amendment C1-a). Our `pct` renderer appends the sign to + the STORED number, so writing the vendor's raw 0.0656 would print a 6.6% creator as `0.0%` — + measured on the Instagram side, and TikTok sends the same shape on all three of its rates. + """ + v = _first(node, *names) + if v is None: + return None + out = engine._pct100(v) + return out or None + + +def _tt_day(node, *names): + """A vendor stamp → `YYYY-MM-DD`, or None. Our `date` columns store a day.""" + v = _first(node, *names) + if v is None: + return None + return engine._day(v) or None + + +def _drop_blanks(row): + """The one place a mapped row loses its `None`s — see the BLANK MEANS NOT READ rule above.""" + return {k: v for k, v in row.items() if v is not None and v != ""} + + +def normalize_profile(node, handle=""): + """A TikTok Profiles row → the `ut_tt_profile` / `ut_tt_snapshots` cell shape. + + ⚠ TWO FIELDS READ THROUGH A CANDIDATE PAIR, and both pairs are the vendor's, not a guess: + * `biography` is PRIMARY and `signature` the FALLBACK — the probe measured `signature` + populated on 85% of rows and they carry the same text; + * `region` is PRIMARY and `country` the FALLBACK — `region` is the one with a documented + two-letter-ISO description, `country` has no description at all. + ⚠ `videos_count` is mapped to `posts_count` with a caveat recorded rather than hidden: its + `ai_description` ranges 1-89, so it may be a WINDOW rather than a lifetime total. It is the + only count of its kind the dataset offers. + """ + node = node if isinstance(node, dict) else {} + account = _tt_str(node, "account_id") or str(handle or "").strip().lstrip("@") + return _drop_blanks({ + "platform": engine.PLATFORM_TIKTOK, + "handle": account, + "full_name": _tt_str(node, "nickname"), + "tt_id": _tt_str(node, "id"), + "profile_url": _tt_str(node, "url") or (tt_profile_url(account) or None), + "bio": _tt_str(node, "biography", "signature"), + "external_url": _bd_first_url(_first(node, "bio_link")), + "verified": _bd_flag(node, "is_verified"), + "is_private": _bd_flag(node, "is_private"), + # ⚠ APPROXIMATE, and the word is the vendor's: `is_commerce_user` has *"many null values"* + # on their own description. It is the closest thing TikTok has to Instagram's + # `is_business_account`, and `_bd_flag` writes nothing at all when the key is absent — so + # the approximation only ever fills a cell the vendor actually answered. + "is_business": _bd_flag(node, "is_commerce_user"), + "followers": _ig_int(_first(node, "followers")), + "following": _ig_int(_first(node, "following")), + "posts_count": _ig_int(_first(node, "videos_count")), + # ⚠ `likes` on a PROFILE row is likes RECEIVED across the account's videos (18-110,200, no + # nulls). It is not a post-level number and it is not our `likes` column, which is why it + # is stored under a different name. + "likes_received": _ig_int(_first(node, "likes")), + "avg_engagement": _tt_pct(node, "awg_engagement_rate"), + "like_engagement": _tt_pct(node, "like_engagement_rate"), + "comment_engagement": _tt_pct(node, "comment_engagement_rate"), + "country_code": _tt_str(node, "region", "country"), + "region": _tt_str(node, "region"), + "predicted_lang": _tt_str(node, "predicted_lang"), + # ⚠ ACCOUNT AGE, NOT A MEASUREMENT STAMP — `create_time` on a profile is when the ACCOUNT + # was made. TikTok stamps nothing with "when this number was true", exactly like Instagram, + # which is why the append law dates a snapshot by when WE read it. + "account_created_at": _tt_day(node, "create_time"), + "source_payload": _bd_source_payload(node), + }) + + +def tt_post_type(node): + """A TikTok post row → one of OUR three type options, or None. + + ⛔ THE FIRST OF THE PROBE DOC'S TWO NAMED BLOCKERS. The vendor's vocabulary is `"video"` / + `"content"`; ours is `image` / `video` / `carousel` and has no `"content"`. Writing the + vendor's token would fail `_clean_field`'s option check on the way in and would put an + untranslated API word in front of a user on the way out. + + So: `video` is `video`, and `content` — TikTok's photo-mode post — is `image`, EXCEPT when the + row carries more than one `carousel_images` entry, which is what a carousel IS on either + network. ⚠ The multi-image branch is decided from the IMAGES, never from the type token: the + token cannot express it, so inferring `carousel` from the word would be inventing a fact. + ⚠ An UNKNOWN token returns None rather than defaulting to `video` (99.5% of rows are video, and + that is exactly what would make the wrong default invisible). + """ + raw = str((node or {}).get("post_type") or "").strip().lower() + if raw == TT_POST_TYPE_VIDEO: + return "video" + if raw != TT_POST_TYPE_CONTENT: + return None + images = (node or {}).get("carousel_images") + return "carousel" if isinstance(images, list) and len(images) > 1 else "image" + + +def normalize_post(node): + """A TikTok Posts row → the `ut_tt_posts` cell shape. None when it carries no identity. + + ⛔ THE SECOND NAMED BLOCKER, RESOLVED HERE AND NOWHERE ELSE: `play_count` is ONE number and + Instagram's schema has TWO columns for it (`plays` and `views`). On TikTok they are the same + fact — `play_count` IS the count TikTok displays under a video — so it maps to `views` and + `ut_tt_posts` HAS NO `plays` COLUMN. Copying one vendor number into two of our columns would + manufacture a second measurement that a rollup could average or double-count, which is a worse + outcome than the missing column it would paper over. + + ⚠ `shortcode` reads `shortcode` then `post_id`: both are 19-digit numerics on this dataset and + the probe measured them as the same shape. That equality is what lets the comments→posts link + join with NO normaliser, which the Instagram side never had. + ⚠ `num_share_count` (a number) is preferred over `share_count` (typed TEXT by the vendor). + ⚠ `commerce_info` is a business/commerce LOCATION per its own description — cities and + countries. It is NOT a paid-partnership flag, and nothing in this dataset is: TikTok declares + no equivalent, so `paid_partnership`/`partner` have no column on this family at all. + """ + node = node if isinstance(node, dict) else {} + shortcode = _tt_str(node, "shortcode", "post_id") + if not shortcode: + return None + return _drop_blanks({ + "platform": engine.PLATFORM_TIKTOK, + "shortcode": shortcode, + # ⛔⛔ `account_id`, NOT `profile_username` — MEASURED on a real Posts row 2026-08-12. + # The vendor's `profile_username` is the DISPLAY NAME (`"Dina"`), while `account_id` is the + # @handle (`"d1na_th"`) — the same field `normalize_profile` already reads for `handle`, so + # one name means one thing across both corpora. Reading the display name silently broke the + # only join this table has: `ut_tt_posts.influencer_key` -> `ut_tt_profile.handle` matched + # NOTHING, so a person could not filter posts by creator and a rollup would count zero. + # ⚠ NO FALLBACK TO `profile_username`, deliberately. It is not a degraded handle, it is a + # different fact, and filling a join key with it is worse than leaving it blank — a blank + # is visibly missing, a display name looks like an answer [[one-question-two-normalizers]]. + # The URL is the honest second source: it carries the handle by construction. + "influencer_key": (_tt_str(node, "account_id") + or tt_handle(_tt_str(node, "profile_url", "url") or "") or None), + "posted_at": _tt_day(node, "create_time"), + "type": tt_post_type(node), + "caption": _tt_str(node, "description"), + "url": _tt_str(node, "url"), + "hashtags": _bd_list(node, "hashtags"), + "tagged_location": _tt_str(node, "commerce_info"), + "views": _ig_int(_first(node, "play_count")), + "likes": _ig_int(_first(node, "digg_count")), + "comments": _ig_int(_first(node, "comment_count")), + "shares": _ig_int(_first(node, "num_share_count")), + "saves": _ig_int(_first(node, "collect_count")), + "video_duration": _ig_int(_first(node, "video_duration")), + "source_payload": _bd_source_payload(node), + }) + + +def normalize_comment(node): + """A TikTok Comments row → the `ut_tt_comments` cell shape. None without a comment id. + + ⚠ THE NAME COLLISION, RESOLVED: TikTok's `replies` is an ARRAY of reply objects and OUR + `replies` column is an INT count. The count comes from `num_replies`; the array stays whole in + `source_payload`. Reading the array's length instead would be a second, disagreeing answer to + a question the vendor already answers — and it would disagree, because a page of replies is not + all of them. + ⚠ `date_created` is typed `date` by this vendor, unlike Instagram's `comment_date` which is + text and needs defensive parsing. It still goes through `_tt_day` — one date path, so a vendor + that changes its mind cannot change ours. + ⚠ The comment TEXT and every identifiable commenter field (`commenter_user_name` is flagged + PII) stay in `source_payload` and are promoted to no column, which is the same posture the + Instagram comment schema takes for the same D-24 reason. + """ + node = node if isinstance(node, dict) else {} + comment_key = _tt_str(node, "comment_id") + if not comment_key: + return None + return _drop_blanks({ + "platform": engine.PLATFORM_TIKTOK, + "comment_key": comment_key, + "shortcode": _tt_str(node, "post_id"), + # ⭐⭐ OWNER RULING 2026-08-12: the comment's CONTENT gets a column. `comment_text` is the + # vendor's own key and `comment_text_only` its stripped variant (the probe recorded both); + # primary first, so a row carrying the rich form is not silently served the plain one. + # ⛔ The commenter's identity is deliberately NOT promoted — see `TT_COMMENT_FIELDS`. + "text": _tt_str(node, "comment_text", "comment_text_only"), + "commented_at": _tt_day(node, "date_created"), + "likes": _ig_int(_first(node, "num_likes")), + "replies": _ig_int(_first(node, "num_replies")), + "source_payload": _bd_source_payload(node), + }) + + +#: ⭐ The three maps, addressable by name — so a gate (and the runners in T04-T06) can walk them +#: rather than naming three functions, and so adding a fourth dataset is one entry. +TT_NORMALIZERS = { + "tt_profile": normalize_profile, + "tt_post_metrics": normalize_post, + "tt_comments": normalize_comment, +} + + +# --------------------------------------------------------------------------------------------- +# THE FETCH — wave 30 · W30-T08 (carrying wave-29's dropped T05) +# --------------------------------------------------------------------------------------------- + +def tt_handle(url): + """A TikTok profile URL **or** a bare handle → the handle. `''` when it is neither. + + Deliberately permissive about the input and strict about the output, because the two callers + hand it different things: an automation stores whatever a person typed in the profile column + (`@nurilab`, `nurilab`, or the full URL), while the discovery runner already holds a clean + `account_id`. One normaliser, so a row found by discovery and a row typed by hand cannot + resolve to two different handles. + """ + s = str(url or "").strip() + if not s: + return "" + if "tiktok.com" in s.lower(): + # Everything after the first `@`, up to the next path segment or query. + tail = s.split("@", 1)[1] if "@" in s else "" + s = tail.split("/")[0].split("?")[0].split("#")[0] + s = s.strip().lstrip("@").strip() + # A handle is the vendor's `account_id` shape: alphanumerics, dots and underscores. + return s if s and all(c.isalnum() or c in "._" for c in s) else "" + + +def tt_post_urls(node, limit=0): + """⭐ WAVE 30 · T10 — the profile row's own post permalinks, newest-first as the vendor sends. + + ⛔ THIS IS WHY TIKTOK POST CAPTURE COSTS NO EXTRA DISCOVERY. `top_videos` rides the PROFILE row + we have already bought, so the posts read is a scrape of links we hold, never a search for them. + The two TikTok DISCOVERY datasets (posts-by-profile, posts-by-keyword) are **404 for our key**, + so a design that reached for either would not merely be dearer, it would not work. + + ⛔⛔ CORRECTED 2026-08-12 — THIS DOCSTRING USED TO SAY *"the probe MEASURED `top_videos` as an + array of video permalinks, NO empties"*, AND THAT SENTENCE IS WHAT SHIPPED THE BUG. The probe + read `/datasets/{id}/metadata` — a DATASET description — and the phrase quoted was the field's + `ai_description`, not an observation of a row. The real row sends **dicts keyed `video_url`** + (measured below), so the reader built against the quoted sentence found nothing, forever, in + silence. ⭐ The transferable half: *"the probe measured X"* and *"the probe read a declaration + of X"* are different claims, and prose cannot be told apart by a reader downstream — which is + why the correction names the method, not just the value. + + ⚠ `top_posts_data` is deliberately NOT read: the probe calls it *"a thin dup of `top_videos`"*, + and preferring whichever happened to be longer is how one creator's window silently differs + from another's. + ⚠ `limit <= 0` means "everything the row carried". The CAP IS THE CALLER'S — `config.maxPosts`, + validated 1..12 — and it is applied here rather than after the scrape so an unwanted post is + never bought. [[a-constant-two-features-share]]: the 12 is the vendor's measured profile window, + not a number this function may invent. + """ + raw = (node or {}).get("top_videos") + out = [] + for item in raw if isinstance(raw, list) else []: + # ⛔⛔ MEASURED ON A REAL ROW 2026-08-12, AND IT IS NOT WHAT THE SCHEMA SAID. + # `top_videos` is NOT an array of permalink strings. One paid TikTok Profiles scrape of a + # live handle returns **19 DICTS**, keyed + # `video_url · video_id · playcount · diggcount · commentcount · share_count · + # favorites_count · create_date · cover_image`. + # The docstring above cites the $0 probe as having "measured" permalinks — it had not, and + # could not: `/datasets/{id}/metadata` describes a DATASET, and the probe's own verdict says + # so in terms (*"what $0 cannot buy … the real ROW shape … that a declared field + # POPULATES"*, `wave29/proto/tiktok-schema.md`). This is the SECOND time this vendor's + # declaration has diverged from its delivery on this exact axis; BD's IG Reels `views` was + # the first. [[reachable-is-not-the-same-as-built]] + # ⇒ The consequence, live: `TT_DS_POSTS` was never reached, because this returned an EMPTY + # list on every real profile — post capture could not have worked for anybody, and the + # T10 gate stayed green because its canned fixture encoded the DECLARED shape. A fixture + # written from a schema tests the schema. + # ⚠ `video_url` is FIRST because it is the key the vendor actually sends; `url` is kept + # because it costs nothing and is what a future corpus revision would most likely use. The + # bare-string branch stays for the same reason — this widens what is ACCEPTED and invents + # nothing: a shape that yields no `http…` value still degrades to "no posts", exactly as + # before, rather than to a URL built out of a guess. + # ⚠ `top_posts_data` is STILL not read (it carries `post_url` and would work): preferring + # whichever array happened to be longer is how one creator's window silently differs from + # another's, and that reasoning is unchanged by this correction. + if isinstance(item, dict): + u = str(item.get("video_url") or item.get("url") or "").strip() + else: + u = str(item or "").strip() + if u.lower().startswith("http") and u not in out: + out.append(u) + return out[:limit] if limit and limit > 0 else out + + +def pull_posts_tt(post_urls, log=print, deferred=None): + """The TikTok Posts dataset for a list of permalinks → `(rows, note)`, already normalised. + + ⚠ ONE CALL FOR THE WHOLE WINDOW. `bd_scrape` has always taken a list, and the Instagram side + measured what happens when a caller forgets: 25 records, one billed snapshot each, a walk still + running at 67 minutes. Nothing here loops per URL. + """ + urls = [str(u) for u in (post_urls or []) if str(u or "").strip()] + if not urls: + return [], "" + rows, note = bd_scrape(TT_DS_POSTS, urls, deferred=deferred) + if note: + log(f"[aios-tt] posts: {note}") + return [], note + out = [r for r in (normalize_post(n) for n in rows) if r] + return out, "" + + +def pull_comments_tt(post_urls, log=print, deferred=None): + """The TikTok Comments dataset for a list of POST permalinks → `(rows, note)`, normalised. + + ⛔ THE MOST EXPENSIVE THING THIS PRODUCT BUYS, and the reason `commentMetrics` defaults OFF on + both networks: a comments scrape ingests identifiable third parties who never entered anybody's + list (D-24). The mapper already keeps every commenter field in `source_payload` and promotes + none of them to a column; this function adds no new exposure, it just has to be asked for. + """ + urls = [str(u) for u in (post_urls or []) if str(u or "").strip()] + if not urls: + return [], "" + rows, note = bd_scrape(TT_DS_COMMENTS, urls, deferred=deferred) + if note: + log(f"[aios-tt] comments: {note}") + return [], note + out = [r for r in (normalize_comment(n) for n in rows) if r] + return out, "" + + +#: ⭐⭐ WAVE 30 · D-156 — THE MEDIA DATASETS, AS A SET, SO THE HAND-OFF CAN FILTER ON IDENTITY. +#: `_media_deferrals` uses this to lift ONLY posts/comments snapshots out of the local deferral +#: list. That is what makes it structurally impossible to file a PROFILE snapshot in the engine's +#: metric queue — the defect a draft of T10 shipped and A-39 booked as "the wrong fix is worse +#: than the gap". A membership test cannot be got wrong by a later edit the way `if` order can. +TT_MEDIA_DATASETS = (TT_DS_POSTS, TT_DS_COMMENTS) + + +def _media_deferrals(deferred): + """The POSTS/COMMENTS entries of a `bd_scrape` deferral list — never the profile's. + + ⚠ The engine, not this module, decides what a deferral MEANS: it stamps `kind` and the + handle and files it. TikTok needs no `_tag_metric_deferrals` twin because one dataset is one + kind here, so the id already carries everything a mapper choice depends on — and importing + Instagram's tagger is not available anyway (W30-T09 gates ZERO `from connectors_ig` lines). + """ + out = [] + for d in deferred or []: + if isinstance(d, dict) and str(d.get("datasetId") or "") in TT_MEDIA_DATASETS: + out.append(dict(d)) + return out + + +def pull_profile_tt(url, log=print, pending_profile=None, prefetch=None, + max_posts=0, post_metrics=False, comment_metrics=False): + """ONE TikTok profile from the vendor. Same return contract as `pull_profile`. + + `{state, profile, posts, comments, via, note}` with `state ∈ ok | partial | blocked | error`, + so the engine's enrich branch treats every network identically and no caller learns a new + shape. + + ⭐ WAVE 30 · T10 — POSTS AND COMMENTS ARE REAL NOW, AND BOTH DEFAULT OFF, exactly as Instagram's + do. `post_metrics` scrapes the profile row's own `top_videos` permalinks (see `tt_post_urls` — + no discovery call, because both TikTok discovery datasets 404 for our key); `comment_metrics` + then scrapes the comments of the posts that came back. ⚠ COMMENTS REQUIRE POSTS by construction + rather than by a rule: their input IS a post permalink, so asking for comments with post capture + off is a request with no subject, and it returns none instead of quietly buying posts nobody + asked for. + + ⛔ `partial` IS THE SUCCESS STATE WHENEVER NO MEDIA WAS READ, and that is deliberate rather than + pessimistic. The Instagram contract reads `ok` only when identity AND media both landed + (`pull_profile_bd`: *"identity without media is still partial ... a run that wrote a follower + count and no posts must not paint green over a posts table that did not grow"*). So: posts not + ASKED for → `partial`, saying so; posts asked for and landed → `ok`; asked for and none came → + `partial` with the vendor's reason. The state answers "did this pull deliver what it went for", + never "did the function finish". + + ⚠ **NO FREE RUNG, AND NO FALLBACK CHAIN.** Instagram's `pull_profile` drops to Apify when the + paid rung refuses; `providers.DEFAULT_CHAINS["tt_profile"]` is deliberately single-provider, + with its own note explaining that a multi-provider chain is a promise something walks it and + that nothing walks Instagram's second name today either. So a refusal here is final, and it + says so instead of implying a retry somewhere. + """ + handle = tt_handle(url) + if not handle: + return {"state": "error", "profile": {}, "posts": [], "comments": [], "via": "", + "note": f"{url!r} is not a TikTok profile URL or handle"} + + # ⭐ THE BATCH FAST PATH, same shape as the Instagram side: `prefetch` is `{handle: node}` from + # one multi-URL scrape covering a whole selection. A hit is a vendor round trip that does not + # happen; a miss falls through to the single-URL call below. + cached = prefetch.get(handle) if isinstance(prefetch, dict) else None + _deferred = [] + if isinstance(cached, dict) and cached: + rows, note = [cached], "" + else: + rows, note = bd_scrape(TT_DS_PROFILES, [tt_profile_url(handle)], deferred=_deferred) + + node = rows[0] if rows else {} + profile = normalize_profile(node, handle) if node else {} + # ⛔ THE READABILITY TEST IS `followers`/`following`, NOT "did we get a dict". `normalize_profile` + # drops blanks, so an unreadable row still returns `{"platform": …, "handle": …}` — truthy, and + # carrying nothing anybody asked for. The Instagram rung tests exactly this pair for exactly + # this reason, and answering "0 followers" instead is the failure it exists to prevent. + unreadable = profile.get("followers") is None and profile.get("following") is None + if note or unreadable: + # ⭐ THE DEFERRAL IS HANDED OVER RATHER THAN DISCARDED. A snapshot the vendor is still + # building HAS ALREADY BEEN PAID FOR; dropping its id bills again on the next run for the + # same record. That was live on the Instagram profile path until 2026-08-09 — measured on + # nurilab as two runs, two fresh snapshots, both abandoned — and it is not being + # reintroduced here by omission. + if isinstance(pending_profile, list): + for d in _deferred: + pending_profile.append({**d, "kind": "profile", "influencer": handle}) + why = note or ("the scrape answered, but no follower/following counts were readable in it " + "(the field names may have moved - see tiktok-capture.md)") + return {"state": "blocked", "profile": {}, "posts": [], "comments": [], "via": "brightdata", + "deferredProfile": [d.get("snapshotId") for d in _deferred], + "note": why} + + # --- W30-T10: THE MEDIA, ONLY WHEN IT WAS ASKED FOR. ------------------------------------ + if not post_metrics: + return {"state": "partial", "profile": profile, "posts": [], "comments": [], + "via": "brightdata", + "note": note or "profile read; post capture is off for this step"} + urls = tt_post_urls(node, limit=max_posts) + if not urls: + # ⚠ NOT AN ERROR AND NOT A RETRY. A creator with no `top_videos` has nothing to buy, and + # saying so is what stops the next run paying to be told the same thing. + return {"state": "partial", "profile": profile, "posts": [], "comments": [], + "via": "brightdata", + "note": note or "profile read; this account's row carried no post links"} + posts, p_note = pull_posts_tt(urls, log=log, deferred=_deferred) + comments, c_note = ([], "") + if comment_metrics and posts: + # The comments dataset is keyed on a POST permalink, so it reads the posts we just bought — + # `url` from the mapper, never the profile's raw array, so a post the posts scrape refused + # is not silently asked about again one rung later. + comments, c_note = pull_comments_tt([p.get("url") for p in posts if p.get("url")], + log=log, deferred=_deferred) + # ⭐⭐ WAVE 30 · D-156 — THE MEDIA DEFERRALS ARE HANDED BACK, and the shape of the hand-off is + # the whole lesson. An earlier draft of T10 appended every `_deferred` entry to + # `pending_profile` tagged `kind: "profile"`. By the time control reaches here a PROFILE + # deferral is impossible — the profile branch above returns `blocked` on any note — so **every + # id fanned out that way was a POSTS or COMMENTS snapshot in the PROFILE queue**, whose + # collector writes preset profile cells onto somebody's record from post rows. The engine keeps + # the two queues apart deliberately (`_pending_profile_tasks` vs `_pending_metric_tasks`). + # ⇒ So this returns them under their OWN key, filtered by dataset identity + # (`_media_deferrals`), and the engine files them in the metric queue with the handle it + # already holds. Returning rather than appending also keeps the queue's vocabulary out of a + # connector: this module knows which CORPUS deferred, never what the engine calls it. + # ⚠ `deferredMedia` rides BOTH returns on purpose. The empty-posts case is the one that + # matters most — that is exactly the run where the vendor took too long, so a caller reading + # the ids only from the success path would lose every batch it actually paid for. + deferred_media = _media_deferrals(_deferred) + if not posts: + return {"state": "partial", "profile": profile, "posts": [], "comments": [], + "via": "brightdata", "deferredMedia": deferred_media, + "note": p_note or note or "profile read; the post source returned nothing"} + return {"state": "ok", "profile": profile, "posts": posts, "comments": comments, + "via": "brightdata", "deferredMedia": deferred_media, + "note": c_note or note or ""} diff --git a/api/main.py b/api/main.py index c1b8f579e86345b31a2c74bfd67637dad0f2e781..b0bf95ce7d8a2b1bc520d88fd08fec2da6d5f0b6 100644 --- a/api/main.py +++ b/api/main.py @@ -1,1081 +1,1081 @@ -"""AIOS web API — the ONE shared, stateless process the Streamlit exit is aimed at. - -It REUSES the `platform/` data layer verbatim (nothing re-implemented): the canonical, -reconciled Odoo model stays server-side and the browser only ever sees derived JSON. Serves the -JSON API under `/api/*` and the built React bundle (`aios-web/web/dist`) at `/`. - -WHAT CHANGED IN EXIT WAVE 1 (2026-07-30) — three things, all of them load-bearing: - - * **Real sessions replace HTTP Basic.** The whole app used to sit behind one shared - `APP_PASSWORD` with the browser's native Basic prompt, which means every caller was the same - anonymous principal and no route could scope anything. Now a signed stateless cookie carries - a real `core/users` identity (X3), and every v1 route resolves BU + own-book scope from the - user RECORD. `/api/health` stays unauthenticated (liveness only). - * **The overlay fork is deleted.** `aios-web/api/data/overlay.json` was a second writable home - for user-owned fields the Streamlit app keeps in the tenant store. ONE store now (C1c). - * **The SSL shim is env-gated.** It used to run at import, unconditionally. - -⚠ TENANT RESOLUTION IS PER REQUEST (X7). Nothing tenant-shaped is held at module level; the -session's tenant claim resolves through `harness.runtime.get_runtime`, which is LRU-bounded. This -is the rule the ~30–80 MB/tenant target depends on — EXIT-0 measured the alternative at a ~0.6 GB -commit FLOOR per tenant, duplicated within 2% per tenant, nothing shared. -""" -import os -import sys -from pathlib import Path - -# --- the local Odoo SSL quirk, now BEHIND A GATE ------------------------------------------------- -# The Windows trust store reports the (valid) Odoo cert as expired, so local runs need an -# unverified default context; the HF Space and the container are unaffected. This used to run -# unconditionally at import — i.e. the shipped container disabled TLS verification for every -# outbound HTTPS call it ever made, to Odoo and to everything else, forever. That is a -# man-in-the-middle away from being someone else's data. It is now opt-in, must be set -# deliberately, and is never on by default. -# ⛔ NEVER set AIOS_INSECURE_SSL in a deployed environment. It exists for one developer laptop. -if os.environ.get("AIOS_INSECURE_SSL") == "1": - import ssl - ssl._create_default_https_context = ssl._create_unverified_context # noqa: S323 - -# --- reuse the tenant #0 data layer verbatim. RI_DIR overrides the location in the container -# (where the layout differs from the local sibling-dir default). --- -_HERE = Path(__file__).resolve() -_RI = Path(os.environ.get("RI_DIR") or (_HERE.parents[2] / "platform")) -for _p in (str(_RI), str(_HERE.parent)): - if _p not in sys.path: - sys.path.insert(0, _p) - -from dotenv import load_dotenv # noqa: E402 -load_dotenv(_RI / ".env") # Odoo creds + APP_PASSWORD, git-ignored, never committed/printed - -from fastapi import Body, Depends, FastAPI, Request # noqa: E402 -from fastapi.middleware.gzip import GZipMiddleware # noqa: E402 -from fastapi.responses import JSONResponse # noqa: E402 -from fastapi.staticfiles import StaticFiles # noqa: E402 -from starlette.exceptions import HTTPException as StarletteHTTPException # noqa: E402 - -import aios_session # noqa: E402 -import routes_admin # noqa: E402 -import routes_alerts # noqa: E402 (wave 20 item 25 — the Alerts inbox) -import routes_assets # noqa: E402 (wave 18 C2-ASSET — catalog product imagery) -import routes_auth # noqa: E402 -import routes_automation # noqa: E402 (wave 18 C4-AUTO — SESSION D's router, mounted by A) -import routes_changes # noqa: E402 (wave 29 item 20 / C6 — F's change token, mounted by A) -import routes_customers # noqa: E402 -import routes_grid # noqa: E402 -import routes_keychain # noqa: E402 (wave 18 C7 — keychain + connectors admin surfaces) -import routes_statements # noqa: E402 (EXIT-6 — the statement sender, off Streamlit) -import routes_nav # noqa: E402 -import routes_pages # noqa: E402 -import routes_platform_admin # noqa: E402 (wave 19 R3/R4 — the Loopable cross-tenant plane) -import routes_products # noqa: E402 (wave 15 C-TOPIC — gated, not yet in the nav) -import routes_records # noqa: E402 -import routes_shares # noqa: E402 (wave 20 R10 — grants for views, folders and databases) -import routes_uploads # noqa: E402 (wave 21 C5 — tabular preview for Select-from-file) -import routes_tables # noqa: E402 (wave 18 C3-UT — user-created databases over the wire) -import routes_connectors # noqa: E402 (wave 23 C11 — the connectors directory; SESSION A's router) -import routes_forms # noqa: E402 (wave 23 C9 — the PUBLIC form door; SESSION D's router) -import routes_templates # noqa: E402 (wave 23 C12 — template apply doors; SESSION E's router) -import routes_odoo_tables # noqa: E402 (wave 27 item 17 — Odoo relational; SESSION E's router) -import routes_connected_tables # noqa: E402 (wave 31 T49/C4 — the source-neutral grid door; D's) -import routes_web_agent # noqa: E402 (wave 31 R10/C5 — the web-browsing agent; E's router, A's line) -import routes_query # noqa: E402 (wave 32 R1/C5 — the Query module; E's router, A's line) -import routes_publish # noqa: E402 (wave 33 R5/C2 — the publish door; C's router, A's line) -import routes_brand # noqa: E402 (wave 33 R4/C2/C6 — connector brand marks; G's router, A's line) -import routes_slack # noqa: E402 (wave 33 R4/C2 — Manage agent + the Slack door; D's router, A's line) -import routes_starred # noqa: E402 (wave 35 R4/C2/C3/C5 — the star; E's router, E's line) -import routes_usage # noqa: E402 (wave 35 R9/C7 — the AI usage meter; E's router, E's line) -import routes_feedback # noqa: E402 (wave 35 R8/C6 — feedback to the operator plane; E's router) -import routes_agent_harness # noqa: E402 (wave 36 R8/C4 — the agent harness store; E's router) -import routes_script_views # noqa: E402 (wave 36 R3/R10/C3 — script Views; E's router) -import routes_geo # noqa: E402 (wave 37 R14/A11 — the keyless map provider seam; D's router) -from core import grid_events # noqa: E402 -# D-315 / D-305 (2026-08-18) — the two store REFUSALS get their own app-level handlers below. -# ⚠ Neither is a `StoreUnavailable` subclass, on purpose: routes that degrade a store outage into a -# session fallback must NOT degrade a refusal, or the user is told a change was saved that was not. -from core import data_binding # noqa: E402 -from core import store # noqa: E402 -from deps import Session, module_gate # noqa: E402 - -_WEB_DIST = _HERE.parents[1] / "web" / "dist" - -# --- THE CANONICAL FIELD CONTRACT: a startup assertion, and the referee gate's subject ---------- -# Every field tags its semantic TYPE and its SOURCE. `source='odoo'` is READ-ONLY (Odoo is never -# written); `source='overlay'` is the editable stratum that lives OUTSIDE Odoo. Loaded from the -# ONE canonical file shared with the embedded host (`platform/aios_grid_fields.json`), so -# embed == standalone by construction. -# -# TWO REASONS THIS IS HERE and not folded into the routes: -# 1. It FAILS LOUDLY at import when the file is missing — which means the deploy or the RI_DIR -# layout is broken, and discovering that from a 500 on the first customer request instead of -# at startup costs a debugging session. (The pre-wave main.py had this guard; the EXIT-2a -# rewrite dropped it and this restores it.) -# 2. `aios-web/verify_fields_contract.py` — the cross-side REFEREE — reads `FIELDS` and -# `_PASSTHROUGH_KEYS` from this module to prove the canonical file, `aios_grid.py` and this -# API have not drifted. The rewrite removed them and the referee went red; retargeting the -# gate would have been the wrong repair ([[gate-can-report-green-on-nothing]]: retarget, do -# not delete — but only when the subject genuinely moved. Here it should not have moved). -# -# ⚠ THE ROUTES SERVE A SUPERSET OF THIS. `routes_customers._payload` derives its field list from -# `aios_grid.fields_from_workspace(ws)`, which is `FIELDS` PLUS the session user's own custom_ and -# measure_ columns — the same list the Streamlit host renders. That is the point: the standalone -# shell now sees the user's own columns instead of the bare base contract. `FIELDS` is the -# canonical FLOOR, asserted below to be exactly what `aios_grid` starts from. -import json as _json_contract # noqa: E402 -_FIELDS_PATH = _RI / "aios_grid_fields.json" -if not _FIELDS_PATH.is_file(): - raise FileNotFoundError( - f"AIOS web API: canonical field contract missing at {_FIELDS_PATH}. Set RI_DIR to the " - "platform root (it also carries the data layer this API imports)." - ) -_contract = _json_contract.loads(_FIELDS_PATH.read_text(encoding="utf-8")) -FIELDS = _contract["fields"] if isinstance(_contract, dict) else _contract -# text/status/select/date pass through untouched; every OTHER odoo field is numeric -> rounded. -# Derived from field TYPE (not a hand-kept key list) so a new text/date field can never be -# wrongly rounded. `select` joined 2026-08-02 (dba) — a choice label rounded would be garbage. -_PASSTHROUGH_KEYS = {f["key"] for f in FIELDS - if f["source"] == "odoo" and f["type"] in ("text", "status", "select", - "date")} - -app = FastAPI(title="AIOS web API") - -# The customers payload measured 1.15 MB of JSON on the live Space, shipped UNCOMPRESSED — with -# the 754 KB bundle behind it, most of "the app is slow" was bytes on the wire. gzip takes the -# payload to ~10–15% of that. minimum_size spares the tiny acks the overhead. -app.add_middleware(GZipMiddleware, minimum_size=1024) - - -@app.exception_handler(StarletteHTTPException) -def _error_shape(request: Request, exc: StarletteHTTPException): - """ONE error shape for every non-2xx (X2): `{"error": {"code", "message"}}`. - - `deps.err` already raises detail in that shape; anything FastAPI raises on its own (a 404, a - 422 from a malformed path param) is wrapped here so a client never has to branch on two - different error bodies. - """ - detail = exc.detail - if isinstance(detail, dict) and "error" in detail: - body = detail - else: - body = {"error": {"code": f"http_{exc.status_code}", "message": str(detail)}} - return JSONResponse(body, status_code=exc.status_code, - headers=getattr(exc, "headers", None)) - - -@app.exception_handler(grid_events.StoreUnavailable) -def _store_unavailable(request: Request, exc: grid_events.StoreUnavailable): - """THE SAFETY NET for "the store is down" → 503, from anywhere. - - ⚠ WHY THIS IS APP-LEVEL AND NOT A `try` PER ROUTE. It was a try per route first, and a - `StoreUnavailable` raised while BUILDING THE PAYLOAD — before the route reached its own - try/except — surfaced as a 500. A store outage is a normal operational state and every route - here touches the store at least twice (the workspace read, then the write), so "remember to - wrap it" is a rule that gets forgotten once and then reports the wrong thing. One handler - means a store outage can only ever be a 503, whichever call raised it. - - The seam raises this only when the caller passed no `fallback_ws` — i.e. exactly on this - adapter, which has no durable session dict to degrade into. A 200 over a write that - evaporated is the failure the whole rule exists to prevent. - """ - return JSONResponse( - {"error": {"code": "store_unavailable", - "message": "the tenant store is unavailable. No change was saved"}}, - status_code=503) - - -@app.exception_handler(data_binding.StoreWriteRefused) -def _store_write_refused(request: Request, exc: data_binding.StoreWriteRefused): - """D-315 — this deployment may not write that store. 503, with the REASON. - - ⛔ IT ECHOES THE EXCEPTION'S OWN MESSAGE, WHERE ITS NEIGHBOUR ABOVE USES A FIXED SENTENCE, and - the difference is the point. "The store is unavailable" is the whole truth about an outage; a - refusal has a cause the operator cannot otherwise discover, because per D-160 the Space - environment cannot be read from outside. A refusal that said only "unavailable" would send - somebody hunting for a network fault that does not exist ([[report-the-cause-before-you-fix-it]]). - The message names the deployment, the store and the fix, and carries no customer data. - - ⛔ AND IT IS ITS OWN HANDLER RATHER THAN A SUBCLASS OF `StoreUnavailable`. Several routes carry - `except StoreUnavailable:` blocks that DEGRADE to a session-scoped fallback workspace — correct - for an outage, catastrophic here, because the user would be told the change was saved. A - distinct type falls through every one of them to this handler. - """ - return JSONResponse( - {"error": {"code": "store_write_refused", - "message": f"this deployment may not write the tenant store. No change was " - f"saved. {exc}"}}, - status_code=503) - - -@app.exception_handler(store.StoreConflict) -def _store_conflict(request: Request, exc: store.StoreConflict): - """D-305 — the document moved under this write more times than it could be rebased. 503. - - ⚠ 503 RATHER THAN 409, DELIBERATELY. A 409 invites the client to resolve a conflict, and there - is nothing here for it to resolve: the store already re-read and re-applied this change up to - `_MAX_REBASE` times before giving up. What the caller needs to know is that the write did not - land and nothing was damaged, which is the same contract as the outage above and the same - retry-later shape. ⛔ The one answer this must never be is 200: reaching this handler means the - only way to have "succeeded" was to overwrite somebody else's work. - """ - return JSONResponse( - {"error": {"code": "store_conflict", - "message": f"somebody else changed this database while your change was being " - f"saved, and it could not be merged. Nothing was saved or " - f"overwritten. Try again. {exc}"}}, - status_code=503) - - -@app.get("/api/health") -def health(): - """Unauthenticated LIVENESS only — it must answer before anyone can sign in, so it may not - reveal anything about the deployment beyond "the process is up". No version, no tenant list, - no config: a health endpoint is the one URL every scanner finds first. - - ⛔ THE VERSION DOES NOT GO HERE, and it was asked to (2026-08-04, when LIVE became a pinned - release and "what is LIVE running" needed an answer). A build identifier tells an unauthenticated - caller exactly which commit's known issues apply. It rides `GET /api/v1/settings` instead, behind - a session — and the authoritative copy is the `VERSION` file in the Space repo, which - `deploy_web.space_version()` reads without needing the app to be up at all.""" - return {"ok": True} - - -app.include_router(routes_auth.router) -app.include_router(routes_nav.router) -app.include_router(routes_customers.router) -app.include_router(routes_products.router) -app.include_router(routes_assets.router) -app.include_router(routes_tables.router) -app.include_router(routes_grid.router) -app.include_router(routes_records.router) -# EXIT wave 2: the Y1 page-data envelope (one route for every ported dashboard) and Y4's user -# administration. `routes_pages` imports `pages`, which lazily imports each `pages_*` builder — so -# a new page is a builder module plus one registry line, and nothing here changes. -app.include_router(routes_pages.router) -app.include_router(routes_admin.router) -app.include_router(routes_automation.router) -app.include_router(routes_keychain.router) -app.include_router(routes_statements.router) -# Wave 19 (owner item 13, R3): the LOOPABLE admin plane — the platform's own cross-tenant view. -# Its own prefix (`/api/v1/platform-admin`), never nested under `/admin`, so there is no path -# ambiguity with `routes_admin`'s `{username}` params and no chance of a tenant-admin route and a -# platform-operator route ever shadowing each other. Every path it declares is gated by -# `core.platform_admin.is_platform_admin`, and `verify_api` enumerates this router to prove it. -app.include_router(routes_platform_admin.router) -# Wave 20 (owner items 25 + 18/23/26): the Alerts inbox and the manage-access surface. Both are -# session-gated rather than admin-gated — an alert is a person's own subscription, and sharing is -# something every user does with their own views/folders/databases. -app.include_router(routes_alerts.router) -app.include_router(routes_shares.router) -app.include_router(routes_uploads.router) -# ⭐ WAVE 23 — THE THREE NEW ROUTERS. Mounted here, above the static catch-all at the bottom of this -# file, because `app.mount("/", _AppStatic(...), html=True)` swallows everything it is reached by: -# a router included AFTER it answers 404 forever while importing fine, type-checking fine and -# passing its own gate. That is the wave-20 declared-but-unmounted shape with a different cause. -# -# ⛔ ALL THREE EXISTED, COMPLETE AND GATED, WITH NO MOUNT until the close-out audit — three finished -# features that would have shipped dead. The workers each posted a mount ask and said they would -# signal "ready" first; C waited for a signal that never came while the files landed anyway. **The -# lesson is not "read the mailbox harder": `verify_api`'s enumeration is what caught it, so the -# control is that the enumeration must NAME every router in this file.** -app.include_router(routes_templates.router) # item 7 / C12 — template registry, session-gated -app.include_router(routes_connectors.router) # item 10 / C11 — the connectors directory -# ⚠ routes_forms is the FIRST NEW PUBLIC DOOR since the `_DEV_FIXTURES` scar (see :257 below). Its -# two paths are DELIBERATELY unauthenticated — a form is filled in by someone with no account — so -# it joins `/api/health`, the automation hook and the tick on the exempt list. It resolves a token -# by scanning tenants with a constant-time compare and answers a uniform 403, so a bad token cannot -# distinguish "no such form" from "not yours", and it never echoes a tenant, table or slug. -app.include_router(routes_forms.router) # item 8 / C9 — the PUBLIC form door -# WAVE 27 item 17 — E's Odoo relational doors. Mounted in the SAME change that E's module landed, -# because the wave-23 scar is exactly this: three finished routers shipped with no include_router -# line — complete, gated, type-clean and 404 for every caller. `verify_api.section_mounts` walks -# `main.app.routes` and pins these two paths by NAME, so an unmounted router now goes RED. -app.include_router(routes_odoo_tables.router) -# ⭐⭐ WAVE 31 · T49 / C4 — THE SOURCE-NEUTRAL DOOR TO THE SAME CAPABILITY. -# `/api/v1/connected-tables/{key}/rows` is an ALIAS: every request lands in -# `routes_odoo_tables.odoo_table_rows`, so "the Odoo path behaves byte-identically" holds by -# CONSTRUCTION rather than by two implementations that agree on the day they were written. R2 puts -# Meta Ads on the same mirror, and a Meta campaign served from a URL with `odoo` in it is a name -# that lies to every network tab, log line and bug report. -# ⚠ MOUNTED IN THE SAME CHANGE AS THE MODULE, per the line above and for the same wave-23 scar. -app.include_router(routes_connected_tables.router) -# ⭐⭐ WAVE 29, item 20 / R11 / contract C6 — F's CHANGE TOKEN. `GET /api/v1/changes?scope=` -# answers "did this bucket change" for ~zero cost (an in-memory counter; ZERO `store.get()` deep -# copies), which is what lets a filtered view pick up a row created in another tab, by an automation -# or by a connector sync without re-downloading the world. -# ⛔ THIS LINE IS THE ARTIFACT THIS PROTOCOL LOSES MOST RELIABLY, AND IT WAS ALREADY LOST ONCE HERE: -# F's router and the CLIENT half both shipped complete, so the poller was calling `/api/v1/changes` -# six times a minute and taking a 404 while every one of F's own gates was green. `verify_api`'s D-48 -# leg caught it (`unmatched: [('/api/v1/changes', 'apiBridge.ts')]`) — a client fetch path with no -# mounted route — which is precisely the control the wave-23 scar above was written to install. -# ⚠ Mounted is NOT callable: `section_changes_callable` in `verify_api.py` SIGNS IN and CALLS this -# route, because a route can be mounted and still raise before its own `try:` (D-107's plain-text 500). -app.include_router(routes_changes.router) # item 20 / C6 — F's router, A's line -# ⛔ WAVE 31 (R10 / C5) — E's router, taken by the INTEGRATOR under W31-T08's own done-when -# ("no router ships unmounted") because `main.py` is D's fence and D's queue did not reach it. -# It was written, gated and 404-dead: `verify_web_agent.py` asserts THIS LINE and was red at -# 60/61 for it. Four waves of the same defect — three routers in wave 23, four features in -# wave 29 — is why the assertion exists and why the line is not left for later. -app.include_router(routes_web_agent.router) # R10 / C5 — E's router, A's line -# ⭐⭐ WAVE 32 (R1 / C5, cross-fence wiring 6) — THE QUERY MODULE. E's router, A's line, and the -# FIFTH consecutive wave in which this exact line is the artifact the protocol nearly loses. -# ⛔ MOUNTED HERE, IN THIS BLOCK, AND NOT AT THE END OF THE FILE — measured by SESSION E in its own -# gate before it front-inserted: `include_router` APPENDS, and `app.mount("/", _AppStatic(...), -# html=True)` swallows everything reached after it, so a router added below that mount answers -# **405 on POST and 404 on GET** while every one of its own tests passes. The comment at :213 states -# the rule; E's measurement is what turns it from advice into a number. -# ⚠ `verify_api` asserts `/api/v1/query` in `app.openapi()["paths"]` — NEVER `{r.path for r in -# app.routes}`, which finds nothing in this app because FastAPI wraps included routers (W31). -app.include_router(routes_query.router) # R1 / C5 — E's router, A's line -# ⭐⭐ WAVE 33 (C2) — THE SIXTH CONSECUTIVE WAVE IN WHICH THIS BLOCK IS THE THING THE PROTOCOL LOSES. -# Contract C2 is written for exactly that: a lane creating a `routes_*.py` posts an ASK, and the -# INTEGRATOR adds the line in the SAME wave. Both of these arrived that way (C's `ASK C-1`, G's -# `ASK G-1`), and `verify_api::section_w23_mounts` asserts each path in `app.openapi()["paths"]` -# with an NC that comments a mount out and goes RED. -# ⚠ Same placement rule as the line above — ABOVE the `app.mount("/", _AppStatic(...), html=True)` -# at the end of the file, never after it. -# ⭐⭐ THE PUBLISH DOOR IS MOUNTED AGAIN (2026-08-15). It was withheld for the wave-33 deploy -# because QA reproduced three HIGH defects that are armed ONLY by mounting it. All three are fixed -# in `routes_publish.py`, each at its cause rather than at its symptom: -# 1. W33-T68 — `form` LEFT `PUBLISHABLE_MODES`. A form view's rows ARE the submissions people -# sent it, so publishing one served other people's answers to anyone holding the link. A form -# still has its own public door (`#/form/`) which serves the BLANK form and never rows. -# 2. W33-T69 — `_visible_keys` returns `[]` when a view STORED a `visible` list and none of its -# keys survive, instead of falling back to the table default. The fallback still applies to a -# view that never stored one, which is what it was written for. -# 3. W33-T70 — three separate leaks closed: the rate limit no longer keys on the caller-supplied -# `x-forwarded-for` (it keys on the socket peer and counts FAILURES only, so a shared proxy -# peer cannot become one global bucket); every failing path now spends the same PBKDF2 the -# success path spends, closing the 206x timing gap; and an unknown token on the sibling GET -# answers the LOCKED shape rather than a 403, so that route stops sorting real tokens from -# fake ones for free. -# ⚠ `verify_api::section_w23_mounts` asserts this path in `app.openapi()["paths"]`, so its four -# EXPECTED reds should now go GREEN. A red here after this line means the mount broke, not the gate. -app.include_router(routes_publish.router) # R5 / C2 — C's router, A's line (the publish door) -# ⛔ NOT BEHIND `module_gate("product_data")`, WHICH IS THE WHOLE REASON IT IS A SECOND ASSET DOOR. -# `routes_assets.py` gates EVERY one of its routes on that module, so a connector logo served from -# there would 403 for any account without the product-data grant — i.e. the Connectors directory -# would lose its logos for exactly the accounts most likely to be setting a connector up (C6). -app.include_router(routes_brand.router) # R4 / C2 / C6 — G's router, A's line (brand marks) -# ⛔ TWO OF ITS SIX PATHS ARE UNAUTHENTICATED BY DESIGN (`/slack/events`, `/slack/interact`) and -# that is the DOOR, not an omission — Slack posts to them with no session and could not carry one. -# They are built on `routes_forms`' proven public shape: signing-secret verification, a sliding-window -# rate limit, a body cap and ONE non-oracle 403, so a bad signature cannot distinguish "no such -# workspace" from "not yours". ⚠ There is no auth middleware and no exempt-path allow-list in this -# file to register them in: "public" here IS the absence of `Depends(require_session)`, which is why -# `verify_api` asserts the absence rather than an entry in a list that does not exist. -app.include_router(routes_slack.router) # R4 / C2 — D's router, A's line (Manage agent + Slack) -# ⭐⭐ WAVE 35 (R4/R5/R7, contracts C2/C3/C5) — THE STAR. Mounted in the SAME change that created -# `routes_starred.py`, which is the seventh consecutive wave in which this block is the artefact the -# protocol nearly loses — and the first in which the router's own lane also owns `main.py`, so there -# is no ask/mount pair to drop. `W35-T46` asserts every one of this wave's paths in -# `app.openapi()["paths"]` and CALLS one route from each. -# ⚠ Same placement rule as every line above: ABOVE `app.mount("/", _AppStatic(...), html=True)`, or -# the router answers 404 on GET and 405 on POST while every one of its own tests passes. -app.include_router(routes_starred.router) # R4 / C2 — the star, counts, and record stars -# ⛔ R9 SAYS THIS ROUTE REPORTS AND NEVER ENFORCES, so mounting it cannot cut anybody off — there is -# no ceiling anywhere behind it (`usage_ledger` has no refusal in it). The one AI limit the product -# enforces is per COLUMN and is unchanged (`ai_enrich.ceiling_report`). -app.include_router(routes_usage.router) # R9 / C7 — GET /usage, the one AI meter -# ⛔ ITS TWO DOORS CARRY DIFFERENT WALLS ON PURPOSE (R8, and D-221 is the booked precedent): the -# POST is any authenticated session's own act, the GET is `is_platform_admin` only. Mounting it does -# not widen anything a tenant admin can reach — `verify_api` proves that by having one try. -app.include_router(routes_feedback.router) # R8 / C6 — feedback to the operator plane -# ⭐⭐ WAVE 36 (R8 / C4) — THE AGENT HARNESS FILE STORE, mounted in the SAME change that created -# `routes_agent_harness.py`. Eighth consecutive wave in which this block is the artefact the -# protocol nearly loses; `verify_web_agent::section_w36_harness` asserts this path in -# `app.openapi()["paths"]` and CALLS the route, because mounted is not callable (D-107). -# ⚠ Placement, as for every line above: ABOVE `app.mount("/", _AppStatic(...), html=True)` at the -# end of this file, or a GET answers 404 and a PUT answers 405 while every gate stays green. -# ⛔ ITS PATHS SIT UNDER `/agents/{id}/...`, WHICH `routes_slack` ALSO SERVES — and that is safe -# rather than lucky: a FastAPI path parameter never spans a `/`, so `/agents/{agent_id}` cannot -# match `/agents/x/harness`. The two routers share a prefix and no route. -app.include_router(routes_agent_harness.router) # R8 / C4 — versioned agent harness files -# ⭐⭐ WAVE 36 (R3 / R10 / C3) — THE SCRIPT VIEW, owner item 6. Mounted in the SAME change that -# created `routes_script_views.py`; `verify_script_views.py` asserts both of its paths in -# `app.openapi()["paths"]` AND calls them, and its NC comments this line out. -# ⛔ ITS RUN DOOR SPAWNS A SUBPROCESS AND IS A PLAIN `def`, so FastAPI runs it in the threadpool. -# Mounting it does not put a ten-second wait anywhere near the event loop; the router's own header -# says why that is not a style choice. -app.include_router(routes_script_views.router) # R3 / R10 / C3 — code-script database Views -# ⭐⭐ WAVE 37 (R14, amendment A11) — THE KEYLESS MAP PROVIDER SEAM, mounted on `ASK D-1` in the -# same wave that created `routes_geo.py`. NINTH consecutive wave in which this block is the -# artefact the protocol nearly loses, and the first in which the ask that carries it was itself -# lost: D raised it inside a ``` fence, so `wave_board.py` blanked it and the bus never delivered -# it (D-366). The integrator found it reading the peer mailbox by hand. -# ⚠ Placement, as for every line above: ABOVE `app.mount("/", _AppStatic(...), html=True)` at the -# end of this file, or GET answers 404 and POST answers 405 while every one of its own gates is -# green. D is adding a `web_map` leg that reads THIS FILE and asserts this line — keep it. -# ⛔ R14 put the provider behind a seam precisely so swapping Google in later is a config change: -# tiles = OpenStreetMap · routing = OSRM · geocoding = Nominatim, none of them keyed, all of them -# rate-limited by POLITENESS rather than by a bill (~1 req/s, and bulk harvesting is forbidden). -app.include_router(routes_geo.router) # R14 / A11 — D's router, A's line (map providers) - - -# --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working) -# They are the v1 handlers with the v1 session requirement — NOT the old unauthenticated Basic -# behavior. An alias that kept the old auth would be a bypass of everything above it. -_GATE = module_gate(routes_customers.MODULE) - - -@app.get("/api/customers", deprecated=True) -def _customers_alias(session: Session = Depends(_GATE)): - return routes_customers._payload(session) - - -@app.patch("/api/customers/{pid}", deprecated=True) -def _patch_alias(pid: int, body: dict = Body(default=None), - session: Session = Depends(_GATE)): - return routes_customers.patch_customer(pid, body, session) - - -def _assert_contract_floor(): - """The canonical FIELDS must be exactly what `aios_grid` starts an empty workspace from. - - This is what stops `FIELDS` becoming a constant that exists only to satisfy a gate. If the - canonical JSON and `aios_grid.fields_from_workspace({})` ever disagree, the standalone API and - the embedded host are serving two different schemas and the grid's own contract has forked — - the exact drift `verify_fields_contract.py` was written to catch, now also caught at startup - on whatever machine is actually running. - """ - import aios_grid - base = [f for f in aios_grid.fields_from_workspace({}) if not f.get("custom")] - if [f["key"] for f in base] != [f["key"] for f in FIELDS]: - raise RuntimeError( - "AIOS web API: the canonical field contract and aios_grid.fields_from_workspace({}) " - "disagree on the base field set. Embed and standalone would serve different schemas. " - "Run aios-web/verify_fields_contract.py.") - - -def _startup_notes(): - """Say the two things an operator must know, once, at import — never in a response body.""" - if aios_session.EPHEMERAL_SECRET: - print("[aios-api] AIOS_SESSION_SECRET is not set. Signing with a random per-process " - "key. Sessions will not survive a restart and will not work across workers. " - "Set it in production.") - if os.environ.get("AIOS_INSECURE_SSL") == "1": - print("[aios-api] AIOS_INSECURE_SSL=1. TLS verification is DISABLED for outbound " - "requests. Local development only; never in a deployed environment.") - - -_assert_contract_floor() -_startup_notes() - - -# ⛔ THE STATIC MOUNT IS NOW UNAUTHENTICATED, and that is a change this wave made on purpose. -# Before EXIT-3a, `BasicAuth` middleware gated the WHOLE app including this mount. A branded login -# page cannot live behind a password prompt, so the shell's own assets must be public — which they -# are: `index.html`, the JS/CSS bundle and the favicon reveal nothing. -# -# ⚠ WHAT THAT SILENTLY DE-GATED, caught in review rather than in production. `web/dist/` also -# carries `sample_customers.json`, a DEV FIXTURE copied from `web/public/`. Its 8 customers are -# synthetic, but the `agent` column holds REAL EMPLOYEE NAMES, and it went from Basic-gated to -# publicly fetchable in this commit. Nothing needs it: both the API bridge and `useCustomerData` -# deleted their sample fallback on purpose ("NOTHING HERE FALLS BACK TO sample_customers.json"), -# and it survives only because it sits in `web/public/`. So it is refused here — 404, the same -# answer as any other path that is not part of the app. -# -# The right long-term fix is deleting it from `web/public/` (S2's lane — flagged in the mailbox); -# this guard is what makes the API safe regardless of what the bundle happens to contain. -_DEV_FIXTURES = {"sample_customers.json"} - - -class _AppStatic(StaticFiles): - async def get_response(self, path, scope): - if Path(path).name in _DEV_FIXTURES: - raise StarletteHTTPException(status_code=404, detail="Not Found") - resp = await super().get_response(path, scope) - # Vite content-hashes everything under assets/ (a change is a NEW url), so those are - # immutable — a repeat visit re-downloads zero bytes instead of the whole 750 KB bundle. - # index.html must stay revalidated or a deploy would strand returning browsers on the old - # bundle; ETag/304 makes that revalidation a header exchange, not a transfer. - # ⚠ Normalised first: on a Windows host StaticFiles hands this path with backslashes, - # and `startswith("assets/")` silently skipped every asset (measured on the local probe). - if path.replace("\\", "/").lstrip("/").startswith("assets/"): - resp.headers["Cache-Control"] = "public, max-age=31536000, immutable" - else: - resp.headers["Cache-Control"] = "no-cache" - return resp - - -# static bundle LAST so /api/* wins; html=True serves index.html at / plus the built assets -if _WEB_DIST.is_dir(): - app.mount("/", _AppStatic(directory=str(_WEB_DIST), html=True), name="web") - - -# --- boot prewarm (AIOS_PREWARM=1 — set by the Dockerfile, never by tests) ----------------------- -# Without this, the first visitor after every deploy/restart pays the full Odoo pool build in -# their request. The thread warms the CONSOLIDATED scope (None, None) + every registered page's -# default envelope; scoped users still pay their own scope's first build, once. -# Env-gated rather than a startup event so importing `api.main` in a gate (verify_api and friends -# run against fakes) can never fire a live Odoo pull. -def _seed_and_sync_store(): - """Bring the analytical store (harness.datastore) LIVE for this container — the app.py - bootstrap, mirrored (2026-07-31, owner item 1). - - `harness.datastore` powers every measure column/condition, and ONLY app.py used to call - `ensure_seed()` — so on a fresh Space disk this container resolved measures against a store - that never existed and every measure cell served blank. Seeding alone was NOT enough either, - and that was measured live the same day: "datastore seeded in 1.7s" followed by an endless - `api:measure-column: ModelError the data cache is still warming up` — `ready()` demands - EVERY entity at phase 'live', and a seed that predates a newer entity leaves it un-synced - forever in a process with no sync loop. The SYNC SPRINT after the seed is what closes the - write_date gap and backfills anything the seed lacks (app.py:7779's exact pattern, bounded - passes). Fail-quiet throughout: no seed/token/Odoo → the columns stay blank, the rows still - serve. - """ - import time as _t - t0 = _t.time() - try: - from harness import datastore as _ds - if _ds.ensure_seed(): - print(f"[aios-api] datastore seeded in {_t.time() - t0:.1f}s") - # DEDICATED Odoo connection for this thread (the W6 postmortem rule): the sync's - # search_reads must never interleave on the shared client's xmlrpc transport. - import core.odoo as _odoo - try: - _odoo._tlocal.client = _odoo.OdooClient() - except Exception: - pass - res = {} - for i in range(12): - res = _ds.sync_all(log=lambda *a, **k: None) - print(f"[aios-api] datastore sync pass {i + 1}: " - + ", ".join(f"{k}={v.get('phase')}" for k, v in sorted(res.items()))) - if all(v.get("phase") == "live" for v in res.values()): - break - # ⭐ Wave 21 (item 2, "make sure the metrics are correct"): a cursor sync can never see - # a HARD DELETE, and the downloaded seed carries whatever was deleted since it was cut — - # one reconcile pass at boot removes both classes of phantom row before the first - # measure is served. MEASURED 2026-08-05: five deleted sale_order_line rows = $284.25 of - # phantom YTD revenue, stable across re-syncs, zero the moment reconcile ran. - try: - _ds.reconcile_deletes(log=lambda *a, **k: None) - except Exception: - pass - print(f"[aios-api] datastore sync done in {_t.time() - t0:.1f}s " - f"(ready={_ds.ready()})") - # ⭐⭐ 2026-08-09 (wave 28, D-107) — THE RELATIONAL REBUILD RUNS AT BOOT, HERE. - # - # ⛔ IT DID NOT BEFORE, AND NOTHING SAID SO. The rebuild lived only inside - # `_store_resync_loop`, whose very first statement is `sleep(1800)` — so the earliest a - # freshly booted container could spawn the four locked databases was T+30 MINUTES. The - # symptom was read as "the boot path is silent": `/odoo-tables/status` polled every 30 s - # across a ~20-minute window over two boots returned the pre-wave schema on all 20 - # samples. It was not silent, it had not been asked yet. Both halves of D-107 were like - # this — a thing that never ran, mistaken for a thing that ran and failed. - # - # ⚠ WHY HERE AND NOT IN `_prewarm`: the tables are DERIVED FROM THE MIRROR, and this is - # the exact line where the mirror has finished advancing — seed, up to twelve sync passes, - # then the delete reconcile. Calling it from the other thread would race the seed and hit - # either `ro_con()`'s "still warming" RuntimeError or, worse, a HALF-SYNCED mirror, which - # is D-107's own hypothesis 3: a partial population trips `MAX_SHRINK` and the rebuild - # refuses — correctly, but for a reason that reads like a data loss scare. - # ⚠ Same thread on purpose: it is already a daemon and nothing serves requests behind it. - # ⭐⭐ W35-T45 / R11 — TENANT #0'S ODOO CREDENTIAL MOVES ONTO ITS KEYCHAIN, IN THE CONTAINER. - # ⚠ BEFORE the relational rebuild, deliberately: the rebuild resolves its connector through - # `rt.odoo_source()`, so running the migration first means the very next read already goes - # through the keychain branch and a broken migration is visible in THIS boot's log rather - # than in tomorrow's. It is idempotent, so every later boot is one `list_entries` read. - _migrate_env_odoo("boot") - _pull_meta("boot") - _rebuild_odoo_relational("boot") - # ⭐⭐ W32-T07 — owner items 13 and 15, DELIVERED. See `_sweep_automation_schemas`. - # ⚠ AFTER the two above and not before: those advance the mirror and can take minutes, and - # this sweep is unrelated to it — putting it last means a slow Odoo sync cannot delay the - # one thing on this path that fixes a grid the owner has asked about twice. - _sweep_automation_schemas("boot") - except Exception as e: # noqa: BLE001 - print(f"[aios-api] datastore seed/sync skipped: {e}") - - -def _migrate_env_odoo(why): - """⭐⭐ W35-T45 / R11 — move tenant #0's environment Odoo credential onto its keychain. - - ⛔⛔ IN THE CONTAINER, WHICH IS THE WHOLE REASON THIS IS A BOOT LINE AND NOT A SCRIPT. D-195, - measured three times: a developer's CLI write to the tenant store is reverted by the running - Space within a minute (download-modify-upload, last-write-wins) — and the write REPORTS SUCCESS - every time, then a fresh read confirms it, and it is gone by the next poll. A CLI migration would - be a dry run that lies, and what it would lie about here is a credential. - - ⚠ IDEMPOTENT AND SCOPED TO TENANT #0 by `routes_keychain.env_odoo_available`, so on every other - tenant and on every later boot this is one `list_entries` read and a line. - ⚠ FAIL-QUIET: this must never take a boot down. But it is never SILENT — a skip prints its reason, - because "already migrated", "no keychain key on this deployment" and "the env is incomplete" are - three different operator actions and a blank Keychain page cannot tell them apart. - """ - try: - import routes_keychain as _kc_routes - from harness import runtime as _runtime - rep = _kc_routes.migrate_env_odoo(_runtime.get_runtime("royal-imports")) - if rep.get("done"): - print(f"[aios-api] odoo credential migrated onto the keychain ({why}): " - f"entry={rep['entry']} carried_pause={rep['carried_pause']}" - + (f" PROBLEM: {rep['why']}" if rep.get("why") else "")) - else: - print(f"[aios-api] odoo keychain migration skipped ({why}): {rep.get('why')}") - except Exception as e: # noqa: BLE001 - print(f"[aios-api] odoo keychain migration FAILED ({why}): {type(e).__name__}: {e}") - - -def _pull_meta(why): - """Pull Meta Ads into THIS container's mirror, before the relational rebuild reads it. - - ⛔ WHY IT HAS TO HAPPEN HERE AND NOT ON A LAPTOP. The mirror is a FILE that lives beside the - process; the Space's copy is seeded from the HF dataset and knows nothing about a DuckDB on a - developer's box. Running `meta_store --sync` locally populates the local mirror and the LIVE - product stays empty — which is the whole difference between "the loader works" and "the - product has the data". Odoo is already arranged this way (`sync_all` runs in the container); - this is the same arrangement for the second connector. - - ⚠ FAIL-QUIET AND SILENT WHEN THERE IS NOTHING TO DO. No token => no Meta => one line, no - error: a tenant that has not connected Meta is a normal state, and this runs on every boot. - ⚠ The window is deliberately SHORT here (`META_INSIGHTS_DAYS`, default 7 at boot) because boot - is not the place for a 90-day backfill — the resync pass widens it. - """ - try: - from harness import meta_store as _meta - if not _meta.token(): - print(f"[aios-api] meta sync skipped ({why}): no META_ADS_ACCESS_TOKEN in this " - f"deployment - the connector is idle, not broken") - return - # ⛔ PASSED, NOT SET IN THE ENVIRONMENT. `meta_store.INSIGHTS_DAYS` binds at - # IMPORT, so an `os.environ.setdefault` here executed after the module was - # already loaded and changed NOTHING: every boot pulled 90 days instead of 7, - # which is the slow path that trips the per-ad-account rate limit and never - # finishes. A knob read at import cannot be turned by a caller at runtime. - rep = _meta.sync("royal-imports", log=lambda *_a: None, insights_days=7) - for p in rep.get("problems") or []: - print(f"[aios-api] meta sync PROBLEM ({why}): {p}") - print(f"[aios-api] meta sync done ({why}): " - + ", ".join(f"{k}={v['in_mirror']}" for k, v in sorted(rep["tables"].items()))) - except Exception as e: # noqa: BLE001 - print(f"[aios-api] meta sync FAILED ({why}): {type(e).__name__}: {e}") - - -def _rebuild_meta_relational(why, rt): - """Spawn/refresh the `ut_meta_*` locked databases off the SAME mirror the Odoo half just used. - - ⭐ CALLED FROM INSIDE `_rebuild_odoo_relational`, ON PURPOSE, and the reason is D-29 rather than - tidiness: `harness/datastore` is a ONE-FILE-AT-A-TIME process global, and that caller has just - established which tenant's file this process holds open. Spawning Meta here inherits that - binding instead of rebinding it under live readers — which is the documented way to serve one - tenant's rows to another with nothing raised. - - ⚠ SILENT WHEN THERE IS NOTHING TO DO. A tenant that never connected Meta has no `meta_*` tables - in its mirror; `refresh` returns `{}` and says so once. That is a normal state, not a failure, - and it must not print an error every 30 minutes for every tenant that does not use Meta. - ⛔ Its own try/except for the reason the two passes above have theirs: a Meta refusal must not - cancel an Odoo rebuild that already succeeded. - """ - try: - import meta_relational as _meta - counts = _meta.refresh(rt, why) - if counts: - print(f"[aios-api] meta relational rebuild done ({why}): " - + ", ".join(f"{k}={v}" for k, v in sorted(counts.items()))) - try: - import automation_engine as _engine - _engine.refresh_relations(rt, log=lambda *_a: None) - except Exception as e: # noqa: BLE001 - print(f"[aios-api] meta relation cells failed ({why}): {type(e).__name__}: {e}") - except Exception as e: # noqa: BLE001 - print(f"[aios-api] meta relational rebuild FAILED ({why}): {type(e).__name__}: {e}") - - -def _sweep_automation_schemas(why): - """⛔⛔ WAVE 32 · `W32-T07` — MAKE D'S DECLARATIONS REACH TENANTS THAT ALREADY HAVE THE TABLES. - - Owner items 13 and 15 — the TikTok comments lock, and the comment CONTENT column he says he has - asked for *"many times"*. **Both were already correct in the source.** `TT_COMMENT_FIELDS` - carries `field_def("text", "Comment")` and `TT_LOCKED_TABLES` already contains the comments - table, both since wave 31, with his words quoted in the comment beside them. So this ticket is - not a schema change and there is nothing to design: it is DELIVERY. - - ⛔ THE MECHANISM, WHICH IS THE WHOLE OF IT. `ut_ensure` MERGES fields into an existing table and - stamps `recordMode` — but only **when something calls it**, and the only callers are automation - runs. A tenant whose `ut_tt_comments` was spawned before the declaration changed keeps the old - shape until an automation happens to run against it. Nothing sweeps existing tenants. That is - [[a-migration-that-runs-on-the-next-write]], and it is this wave's stated thesis: a declaration - that never reaches a tenant is indistinguishable from one that was never written. - - ⛔⛔ AND IT MUST RUN **IN THE CONTAINER**, WHICH IS WHY THIS IS IN `main.py` AND NOT A SCRIPT. - D-195, measured three times: a developer's CLI write to the tenant store is reverted by the - running Space within a minute (download-modify-upload, last-write-wins) — and **the write - reports success every time**, then a FRESH read confirms it, and it is gone by the next poll. - A connector's tables must be spawned BY THE CONTAINER; a CLI spawn is a dry run that lies. - - ⚠ EVERY TENANT, unlike `_rebuild_odoo_relational` below — and the asymmetry is deliberate - rather than an oversight. That function is scoped to royal because it derives from the DuckDB - mirror, and `harness/datastore` is a ONE-FILE-AT-A-TIME process global (D-29): rebinding it per - tenant in a daemon thread can serve one tenant's rows to another with nothing raised. This - sweep touches only `user_tables` through each tenant's own `rt`, which has no such global — so - the hazard that scopes that one does not exist here, and TikTok automations run in tenants - other than #0. - - ⚠ CHEAP ON A CORRECT TENANT: `ut_ensure` short-circuits when nothing changed, so this is a read - per child table on a tenant that is already right, and the whole delivery on one that is not. - ⚠ ONE TENANT'S FAILURE MUST NOT STOP THE NEXT. Each is wrapped: a tenant whose store is - unreachable at boot is reported and skipped, never allowed to abort the sweep for everyone. - """ - try: - import automation_engine as _eng - from harness import runtime as _runtime - except Exception as e: # noqa: BLE001 - print(f"[aios-api] schema sweep ({why}) SKIPPED. Import failed: {e}") - return - try: - schemas = _eng.platform_schemas() - except Exception as e: # noqa: BLE001 - print(f"[aios-api] schema sweep ({why}) SKIPPED. No declarations: {e}") - return - tenants = [] - try: - tenants = _runtime.known_tenants() - except Exception as e: # noqa: BLE001 - print(f"[aios-api] schema sweep ({why}) SKIPPED. Tenant list unreadable: {e}") - return - for slug in tenants: - try: - rt = _runtime.get_runtime(slug) - # ⛔⛔ FIXED 2026-08-13 — THIS SWEEP WAS SPAWNING EIGHT DATABASES IN EVERY TENANT. - # Owner: *"Database for Royal Imports, why we have fucking IG and TIktok databases."* - # `ut_ensure`'s first line is *"Create the table if it is missing"*, and this loop fed - # it every child of every platform schema for every tenant — so tenant #0, a floral and - # giftware importer with ZERO automations, woke up on 2026-08-13 at 12:05 UTC owning - # `ut_ig_posts`, `ut_ig_comments`, `ut_ig_snapshots`, `ut_ig_post_snapshots` and the - # four TikTok twins, all empty, all in his nav flyout. **The bug is one word wide:** - # this function's own title says *"MAKE D'S DECLARATIONS REACH TENANTS THAT ALREADY HAVE - # THE TABLES"* and its body called a CREATE-OR-MERGE function to do a MERGE-ONLY job. - # [[reuse-and-delete-are-hypotheses]] — `ut_ensure` was the right function for the - # merge and brought a second behaviour nobody wanted with it. - # - # ⭐ THE PREDICATE IS "DOES THIS TENANT ALREADY HAVE THE TABLE", read ONCE per tenant - # rather than per child — `rt.get` deep-copies the whole tenant document (28.6 MB on - # tenant #0), so asking eight times is eight copies to answer one question. - # ⚠ AND IT MUST NOT WEAKEN THE DELIVERY: a tenant that HAS `ut_tt_comments` still gets - # the `text` column and the `recordMode` stamp, which is the entire point of T07. The - # sweep now delivers to tables that exist and mints none, which is what it always - # claimed to do. - have = set(rt.get("user_tables") or {}) - ensured, skipped = [], [] - for s in schemas: - for key, child in (s.get("children") or {}).items(): - if key not in have: - skipped.append(key) - continue - got = _eng.ut_ensure(rt, child["label"], child["fields"], "automation", - key=key, lock_fields=True, - record_mode=child["record_mode"]) - if got: - ensured.append(got) - if skipped: - # ⭐ SAID OUT LOUD, never silently skipped — this repo's "no silent caps" rule. A - # sweep that quietly does nothing looks identical to a sweep that is not running, - # which is how the previous behaviour survived review in the first place. - print(f"[aios-api] schema sweep ({why}) {slug}: {len(skipped)} child table(s) not " - f"present in this tenant, so nothing was created for them " - f"({', '.join(sorted(skipped))}). They are spawned by an automation that " - f"needs them, never by this sweep") - # ⭐ THE RETRACTION (D's `retract_foreign_presets`, D-152), AFTER the children loop. - # ⛔ THE SWEEP ABOVE MAKES COLUMNS **ARRIVE** AND CANNOT MAKE STALE ONES **LEAVE**, and - # T07's `done-when` asserts both ("no `ut_tt_*` grid carries an Instagram column"). On a - # tenant that ran TikTok before W30-T08 the 26 machine-authored IG columns are still - # there — the detector was fixed, the damage never was. - # ⚠ `kept` IS THE HONEST HALF: a foreign column that HOLDS DATA is REPORTED, never - # deleted. If it is non-empty, the screenshot shows a column and the report is the - # answer (W30/R6's second sentence). - st = {} - try: - st = _eng.retract_foreign_presets(rt, log=lambda *a, **k: None) or {} - except Exception as e: # noqa: BLE001 - print(f"[aios-api] schema sweep ({why}) {slug}: retraction FAILED: {e}") - print(f"[aios-api] schema sweep ({why}) {slug}: ensured={len(ensured)} " - f"tables={st.get('tables', 0)} columns={st.get('columns', 0)} " - f"cells={st.get('cells', 0)} flags={st.get('flags', 0)} " - f"kept={st.get('kept') or []}") - except Exception as e: # noqa: BLE001 - print(f"[aios-api] schema sweep ({why}) {slug}: FAILED: {e}") - # ⭐ A SUCCESS MARKER, for `_rebuild_odoo_relational`'s stated reason: D-107 was chased for a - # day on ABSENT log markers, which cannot tell "it ran and was fine" from "it was never - # reached". Three failure markers and no success marker makes silence ambiguous. - print(f"[aios-api] schema sweep ({why}) done over {len(tenants)} tenant(s)") - - -def _rebuild_odoo_relational(why): - """Spawn/refresh the four locked Odoo databases, then compute their cells. `why` is 'boot' or - 'resync' and rides every log line, because "it failed" and "it failed at boot, before anyone - could have asked" are different diagnoses. - - ⭐ THE SUCCESS LINE IS NOT DECORATION — it is the control this path lacked. D-107 was chased - for a day on the strength of *absent* log markers, which cannot distinguish "the rebuild ran - and was fine" from "the rebuild was never reached". Three failure markers and no success - marker means silence is ambiguous; now it is not. - - ⚠ SCOPED TO ROYAL-IMPORTS, deliberately, and it is NOT the D-29 shortcut it resembles. The - caller has just advanced whichever DuckDB file this process holds open — tenant #0's — and - royal is the only tenant with an Odoo mirror to derive from (R1). Iterating tenants here walks - straight into D-29's documented hazard: `harness/datastore` is a ONE-FILE-AT-A-TIME - process-global, and rebinding it in a daemon thread under live readers can serve one tenant's - rows to another with nothing raised. `is_royal` stays the authority on which slugs qualify. - """ - try: - import odoo_relational as _rel - from harness import runtime as _runtime - if not _rel.is_royal("royal-imports"): - return - _rt = _runtime.get_runtime("royal-imports") - counts = _rel.refresh(_rt, "royal-imports") - # ⭐⭐ AND THEN COMPUTE THE CELLS, which `refresh` does NOT do. - # - # ⛔ THE FAILURE THIS CLOSES IS THE WORST-LOOKING KIND. `refresh` writes rows and field - # DEFINITIONS; every Link and Rollup cell comes from a separate pass. Those passes used to - # live only on the automation `tick`, which fires from an EXTERNAL EventBridge cron — so a - # fresh boot landed 71,954 rows with eleven fully-configured relational columns and every - # one of them BLANK until an unrelated scheduler happened to run. Nothing errors; the - # tables simply look finished and answer nothing. - # - # ⚠ `refresh_relations`, NOT `compute_relation_cells`. The latter computes a change count - # over a blob it was handed and PERSISTS NOTHING; the former walks the tenant and writes. - # Calling the inner one here would return a plausible number and change no cell. - # - # ⚠ ONE try/except PER PASS, for the reason `tick` states at its own copies: a source - # rollup REFUSES loudly on a truncated group set, and a shared block would let that honest - # refusal silently cancel a relational pass that had already succeeded. - try: - import automation_engine as _engine - _engine.refresh_relations(_rt, log=lambda *_a: None) - except Exception as e: # noqa: BLE001 - print(f"[aios-api] odoo relation cells failed ({why}): {type(e).__name__}: {e}") - try: - import rollup_sql as _rollup - for _b, _key, _l, _f in _rel.TABLES: - _rollup.compute(_rt, _key) - except Exception as e: # noqa: BLE001 - print(f"[aios-api] odoo source rollups failed ({why}): {type(e).__name__}: {e}") - print(f"[aios-api] odoo relational rebuild done ({why}): " - + ", ".join(f"{k}={v}" for k, v in sorted((counts or {}).items()))) - _rebuild_meta_relational(why, _rt) - except Exception as e: # noqa: BLE001 - # ⚠ The TYPE is named here as it is in the two inner handlers. The original printed only - # `{e}`, and a bare message is exactly what made D-107 unreadable from the outside: a - # `BinderException` about a missing column is a different action from a timeout or an auth - # failure, and the text alone often does not say which it was. - print(f"[aios-api] odoo relational rebuild failed ({why}): {type(e).__name__}: {e}") - - -def _store_resync_loop(): - """Keep the analytical mirror current — the API-process stand-in for the re-sync the - Streamlit app piggybacks on page renders. Measure memos key on the pool stamp, so a - refreshed pool re-reads the freshly synced store.""" - import time as _t - passes = 0 - while True: - # ⭐⭐ WAVE 32 · OWNER ITEM 11 / R11 — THE TENANT'S OWN CADENCE, NOT A HARDCODED 1800. - # - # ⛔ THIS LINE IS WHY THE SETTING WAS NOT A SETTING. SESSION B built the whole of item 11 — - # the 30m/1h/4h/daily/manual presets, the server-side clamp, the config door and - # `odoo_relational.sync_seconds()` which turns the stored preset into seconds — and its own - # ticket said the one-line change belonged to A because `main.py` is A's file. That ASK was - # never sent, so the value was stored, displayed, clamped and IGNORED: a person could pick - # "every 4 hours" and the loop would keep resyncing every 30 minutes with nothing anywhere - # reporting the disagreement. Found by `verify_reachability` naming `sync_seconds` as a - # function whose ONLY caller was its own gate [[artifact-with-no-importer]]. - # - # ⚠ `None` MEANS MANUAL AND MUST NOT MEAN ZERO. R11's `manual` preset returns None from - # `sync_seconds`; treating that as a falsy interval would spin this loop with no sleep at - # all. It parks at the default cadence instead and simply does no work — the tenant asked - # not to be synced automatically, not for the server to stop breathing. - # ⚠ RE-READ EVERY PASS, deliberately: a cadence changed in Settings takes effect on the - # next cycle rather than at the next container restart, which is what makes it a setting. - # ⚠ FAIL-SAFE TO 1800 — an unreadable config must not become a tight loop. The floor is - # enforced in `sync_seconds` as well as at the write door, for the same reason. - # ⛔⛔ WAVE 34 · W34-T46 / D-246 — THE PARAGRAPH ABOVE DESCRIBED A BRANCH THAT DID NOT - # EXIST, AND THAT IS WHY NOBODY LOOKED. It said `manual` "parks at the default cadence - # instead and simply DOES NO WORK". The first half was true; the second half was not - # implemented anywhere: `_secs is None` set the sleep to 1800 and then fell into an - # UNCONDITIONAL `sync_all()`. So a tenant who chose "manual" was synced every thirty - # minutes exactly like everyone else, with the setting stored, displayed, clamped and - # obeyed by nothing — the identical failure this block's own W32/R11 note is about, one - # layer further in. A comment asserting a fix is why the second reader stops reading. - # ⚠ FIXED, AND THE FIX IS NARROW ON PURPOSE. `manual` is an ODOO connector setting, so it - # skips the Odoo mirror sync and its reconcile and NOTHING ELSE: the Meta pull and the - # relational rebuild below still run, because a tenant asking not to have Odoo polled has - # not asked for the other connector to stop. - # ⚠ AND IT STILL SLEEPS AND STILL LOOPS. The config is re-read every pass, so switching - # back off `manual` takes effect on the next cycle rather than at the next restart. - _every = 1800 - _manual = False - try: - import odoo_relational as _rel_cad - from harness import runtime as _rt_cad - # ⚠ D-245, KNOWN AND NOT FIXED HERE: the slug is hardcoded, so this reads TENANT #0's - # cadence and applies it to a loop that syncs the shared mirror. Correct while one - # tenant has Odoo; wrong the moment a second one does. Out of this ticket's scope and - # named rather than silently inherited. - _secs = _rel_cad.sync_seconds(_rt_cad.get_runtime("royal-imports")) - _manual = _secs is None - _every = 1800 if _secs is None else max(int(_secs), 60) - except Exception: # noqa: BLE001 - pass - _t.sleep(_every) - passes += 1 - try: - from harness import datastore as _ds - import core.odoo as _odoo - try: - _odoo._tlocal.client = _odoo.OdooClient() - except Exception: - pass - # ⚠ A PLAIN BRANCH, NOT AN EARLY EXIT AND NOT AN EXCEPTION. A `raise` here would be - # caught by this block's own `except` and printed as "store resync failed", turning a - # setting a person chose into a recurring error in the log; a `continue` would skip - # the Meta pull and the relational rebuild below, which `manual` says nothing about. - if not _manual: - _ds.sync_all(log=lambda *a, **k: None) - # Wave 21 — every 4th pass (~2h): purge hard-deleted rows the cursor sync cannot - # see (the boot pass's comment has the measured case). Cheap id-sweep per entity; - # without it deleted Odoo lines inflate every sum on the mirror FOREVER. - # ⚠ INSIDE the branch: a reconcile is a pass over the mirror this loop was just - # told not to advance. - if passes % 4 == 0: - _ds.reconcile_deletes(log=lambda *a, **k: None) - except Exception as e: # noqa: BLE001 - print(f"[aios-api] store resync failed: {e}") - # ⭐ WAVE 27 item 17 (E's ASK ->A, resolved): the Odoo RELATIONAL tables are rebuilt - # AFTER the mirror they are derived from, in the same pass and in that order — deriving - # from a mirror this loop is about to advance would publish a worklist one cycle stale - # every single time. - # - # ⛔ OUTSIDE the try/except above, NOT folded into it. A failing relational rebuild must - # not swallow the sync's error message, and — worse the other way — a sync failure must - # not skip a rebuild that had nothing wrong with it. Two independent failures, two - # independent logs. (`_rebuild_odoo_relational` carries its own handlers.) - # - # ⚠ THE REBUILD IS NOT THE FRESHNESS GUARANTEE — the rows carry a visible `_refreshed` - # stamp for that. This loop is a daemon thread whose failure path is a `print` (D-29), so - # "the wiring exists" and "the data is current" are different claims and only the stamp - # can tell a user which one they are looking at. - # - # ⭐ ONE IMPLEMENTATION, TWO CALLERS (wave 28). This block used to be the only copy, which - # is what made the boot path silent for 30 minutes after every restart; it is now the - # SECOND caller of the same function `_seed_and_sync_store` calls at boot. A copy here - # would be a second thing to keep in step, and the two would answer differently on the - # next ruling — the exact shape the Views top-up was just fixed for on the other side of - # this wave. - # ⛔ THE META PULL RIDES THE RESYNC TOO, AND LEAVING IT OUT WAS A REAL GAP — caught by - # reading the deploy's own boot log rather than by any gate. `_pull_meta` was wired into - # `_seed_and_sync_store` ALONE, i.e. it ran exactly once per container, at boot, behind a - # full Odoo sync. So a boot where the Graph call was rate-limited, slow or simply after the - # thread died left the mirror empty with NOTHING to retry it: the relational rebuild below - # would then find no `meta_*` tables every 30 minutes forever and skip, silently and - # correctly. A connector that can only ever be established at boot is one bad boot away - # from being permanently absent. - # ⚠ Cheap when there is nothing to do: no token => one line and return. - _pull_meta("resync") - _rebuild_odoo_relational("resync") - - -def _prewarm(): - import time as _t - t0 = _t.time() - # ⚠ The store sync runs in its OWN thread, never ahead of the pool warm: a stale seed can - # take many minutes of XML-RPC to close, and the first live probe of this arrangement - # showed the pool warm (42s) silently queued behind it — the whole app cold for every - # visitor while a background column channel caught up. Measure cells retry via the - # transient rule until the sync lands; nothing else waits on it. - import threading as _th - _th.Thread(target=_seed_and_sync_store, daemon=True, name="store-seed-sync").start() - try: - from harness import runtime as _runtime - rt = _runtime.get_runtime("royal-imports") - routes_customers.warm_default(rt) - import pages as _pages - _pages.warm_default(rt) - # ⭐⭐ WAVE 30 · T12, THE COLD PATH (owner items 4/5). Automation was the ONE module absent - # from this list, so its memo was always filled by a visitor rather than by boot: call 1 of - # `GET /automations` after every deploy downloaded the whole `user_tables` document (35.8 MB - # ceiling, under the store lock) onto whoever clicked first. Memoising the WARM path — two - # waves of it — could not touch that, because the cold call is the one that fills the memo. - # ⚠ It elects a STRING and keeps no document; see `warm_default`'s own note on why caching - # the bucket would trade a latency for memory this tier does not have. - import routes_automation as _rauto - _rauto.warm_default(rt) - print(f"[aios-api] prewarm done in {_t.time() - t0:.1f}s") - except Exception as e: # noqa: BLE001 — boot must not die on a warm-up - print(f"[aios-api] prewarm skipped: {e}") - - -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# ⭐⭐ W31-T46 / D-160 — THE MIRROR IS SEEDED WHETHER OR NOT `AIOS_PREWARM` IS SET. -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# -# THE DEFECT, and it has cost a release once already (owner item 12, wave 20). `_prewarm()` was -# the ONLY caller of `_seed_and_sync_store()`, which is the ONLY caller of -# `datastore.ensure_seed()`. So on a fresh Space disk with `AIOS_PREWARM` anything but `1`: -# no `royal.duckdb` ⇒ `datastore.ready()` False forever ⇒ `ro_con()` refuses ⇒ every measure -# column blank AND — since wave 30 put the Odoo grids on the mirror — **two user-facing grids -# serve nothing**, with the app RUNNING, the deploy green and the tag correct. Last time the -# symptom was read as a pinned tag and a store problem for days. -# -# ⛔ AND THE FLAG IS EASIER TO LOSE THAN IT LOOKS: `deploy_web.py` pushes `AIOS_PREWARM=1` -# EXPLICITLY (to overwrite a stale `0`, because a Space secret survives a redeploy) — but that -# push sits under `if TARGET:`, so an ordinary bare `python deploy_web.py` skips it. A guard -# written for the dangerous case that the ordinary case walks straight past. -# -# ⭐ SO THE SEED IS SPLIT OFF AND MADE UNCONDITIONAL, which is T46's first branch rather than its -# fallback. It is the right half to move because it is the CHEAP, SAFE one: `ensure_seed()` -# touches no Odoo (it is one `hf_hub_download` of a snapshot), returns immediately when the file -# is already there, and returns immediately without `HF_TOKEN`. The EXPENSIVE, live half — -# `sync_all()`'s XML-RPC passes, the pool warm, the resync loop — stays exactly where it was, -# because the env gate's stated reason is still true: importing `api.main` in a gate must never -# fire a live Odoo pull. -# -# ⚠ THE `DB_PATH.exists()` PRE-CHECK IS WHAT KEEPS THIS FREE. It is a stat, and it is False only -# on a genuinely fresh disk — so on every developer box and in every gate run the thread is never -# started at all, and on a fresh Space it does exactly the thing whose absence blanks the grids. -#: What the boot seed did, so a surface can REPORT it instead of an operator inferring it from a -#: blank grid. R6's second sentence: a limit that cannot be removed is reported with its cause. -MIRROR_SEED = {"attempted": False, "seeded": False, "cause": "", "recommendation": ""} - - -def _seed_mirror_if_absent(): - """Hydrate the analytical mirror when this container has none — INDEPENDENT of `AIOS_PREWARM`. - - Fail-quiet by design, and it records WHY rather than only whether: "there is no mirror and no - HF_TOKEN to fetch one" and "there is no mirror and the fetch failed" are different operator - actions, and a blank grid cannot tell them apart. - """ - from harness import datastore as _ds - MIRROR_SEED["attempted"] = True - try: - if _ds.ensure_seed(): - MIRROR_SEED["seeded"] = True - print("[aios-api] analytical mirror seeded at boot (independent of AIOS_PREWARM)") - return True - if not _ds.DB_PATH.exists(): - MIRROR_SEED["cause"] = ( - "this container has no analytical mirror and the seed snapshot could not be " - "fetched (no HF_TOKEN, or the dataset was unreachable)") - MIRROR_SEED["recommendation"] = ( - "set HF_TOKEN on the deployment; until then every connected grid and every " - "measure column served from the mirror is empty") - print(f"[aios-api] NO ANALYTICAL MIRROR: {MIRROR_SEED['cause']}") - except Exception as e: # noqa: BLE001 — boot must not die - MIRROR_SEED["cause"] = f"the boot seed raised {type(e).__name__}: {e}" - MIRROR_SEED["recommendation"] = "check HF_TOKEN and the seed dataset's availability" - print(f"[aios-api] mirror seed skipped: {e}") - return False - - -try: - from harness import datastore as _ds_boot - if not _ds_boot.DB_PATH.exists(): - import threading as _threading_seed - _threading_seed.Thread(target=_seed_mirror_if_absent, daemon=True, - name="mirror-seed").start() -except Exception as e: # noqa: BLE001 - print(f"[aios-api] mirror seed not scheduled: {e}") - -if os.environ.get("AIOS_PREWARM") == "1": - import threading as _threading - _threading.Thread(target=_prewarm, daemon=True, name="prewarm").start() - _threading.Thread(target=_store_resync_loop, daemon=True, name="store-resync").start() +"""AIOS web API — the ONE shared, stateless process the Streamlit exit is aimed at. + +It REUSES the `platform/` data layer verbatim (nothing re-implemented): the canonical, +reconciled Odoo model stays server-side and the browser only ever sees derived JSON. Serves the +JSON API under `/api/*` and the built React bundle (`aios-web/web/dist`) at `/`. + +WHAT CHANGED IN EXIT WAVE 1 (2026-07-30) — three things, all of them load-bearing: + + * **Real sessions replace HTTP Basic.** The whole app used to sit behind one shared + `APP_PASSWORD` with the browser's native Basic prompt, which means every caller was the same + anonymous principal and no route could scope anything. Now a signed stateless cookie carries + a real `core/users` identity (X3), and every v1 route resolves BU + own-book scope from the + user RECORD. `/api/health` stays unauthenticated (liveness only). + * **The overlay fork is deleted.** `aios-web/api/data/overlay.json` was a second writable home + for user-owned fields the Streamlit app keeps in the tenant store. ONE store now (C1c). + * **The SSL shim is env-gated.** It used to run at import, unconditionally. + +⚠ TENANT RESOLUTION IS PER REQUEST (X7). Nothing tenant-shaped is held at module level; the +session's tenant claim resolves through `harness.runtime.get_runtime`, which is LRU-bounded. This +is the rule the ~30–80 MB/tenant target depends on — EXIT-0 measured the alternative at a ~0.6 GB +commit FLOOR per tenant, duplicated within 2% per tenant, nothing shared. +""" +import os +import sys +from pathlib import Path + +# --- the local Odoo SSL quirk, now BEHIND A GATE ------------------------------------------------- +# The Windows trust store reports the (valid) Odoo cert as expired, so local runs need an +# unverified default context; the HF Space and the container are unaffected. This used to run +# unconditionally at import — i.e. the shipped container disabled TLS verification for every +# outbound HTTPS call it ever made, to Odoo and to everything else, forever. That is a +# man-in-the-middle away from being someone else's data. It is now opt-in, must be set +# deliberately, and is never on by default. +# ⛔ NEVER set AIOS_INSECURE_SSL in a deployed environment. It exists for one developer laptop. +if os.environ.get("AIOS_INSECURE_SSL") == "1": + import ssl + ssl._create_default_https_context = ssl._create_unverified_context # noqa: S323 + +# --- reuse the tenant #0 data layer verbatim. RI_DIR overrides the location in the container +# (where the layout differs from the local sibling-dir default). --- +_HERE = Path(__file__).resolve() +_RI = Path(os.environ.get("RI_DIR") or (_HERE.parents[2] / "platform")) +for _p in (str(_RI), str(_HERE.parent)): + if _p not in sys.path: + sys.path.insert(0, _p) + +from dotenv import load_dotenv # noqa: E402 +load_dotenv(_RI / ".env") # Odoo creds + APP_PASSWORD, git-ignored, never committed/printed + +from fastapi import Body, Depends, FastAPI, Request # noqa: E402 +from fastapi.middleware.gzip import GZipMiddleware # noqa: E402 +from fastapi.responses import JSONResponse # noqa: E402 +from fastapi.staticfiles import StaticFiles # noqa: E402 +from starlette.exceptions import HTTPException as StarletteHTTPException # noqa: E402 + +import aios_session # noqa: E402 +import routes_admin # noqa: E402 +import routes_alerts # noqa: E402 (wave 20 item 25 — the Alerts inbox) +import routes_assets # noqa: E402 (wave 18 C2-ASSET — catalog product imagery) +import routes_auth # noqa: E402 +import routes_automation # noqa: E402 (wave 18 C4-AUTO — SESSION D's router, mounted by A) +import routes_changes # noqa: E402 (wave 29 item 20 / C6 — F's change token, mounted by A) +import routes_customers # noqa: E402 +import routes_grid # noqa: E402 +import routes_keychain # noqa: E402 (wave 18 C7 — keychain + connectors admin surfaces) +import routes_statements # noqa: E402 (EXIT-6 — the statement sender, off Streamlit) +import routes_nav # noqa: E402 +import routes_pages # noqa: E402 +import routes_platform_admin # noqa: E402 (wave 19 R3/R4 — the Loopable cross-tenant plane) +import routes_products # noqa: E402 (wave 15 C-TOPIC — gated, not yet in the nav) +import routes_records # noqa: E402 +import routes_shares # noqa: E402 (wave 20 R10 — grants for views, folders and databases) +import routes_uploads # noqa: E402 (wave 21 C5 — tabular preview for Select-from-file) +import routes_tables # noqa: E402 (wave 18 C3-UT — user-created databases over the wire) +import routes_connectors # noqa: E402 (wave 23 C11 — the connectors directory; SESSION A's router) +import routes_forms # noqa: E402 (wave 23 C9 — the PUBLIC form door; SESSION D's router) +import routes_templates # noqa: E402 (wave 23 C12 — template apply doors; SESSION E's router) +import routes_odoo_tables # noqa: E402 (wave 27 item 17 — Odoo relational; SESSION E's router) +import routes_connected_tables # noqa: E402 (wave 31 T49/C4 — the source-neutral grid door; D's) +import routes_web_agent # noqa: E402 (wave 31 R10/C5 — the web-browsing agent; E's router, A's line) +import routes_query # noqa: E402 (wave 32 R1/C5 — the Query module; E's router, A's line) +import routes_publish # noqa: E402 (wave 33 R5/C2 — the publish door; C's router, A's line) +import routes_brand # noqa: E402 (wave 33 R4/C2/C6 — connector brand marks; G's router, A's line) +import routes_slack # noqa: E402 (wave 33 R4/C2 — Manage agent + the Slack door; D's router, A's line) +import routes_starred # noqa: E402 (wave 35 R4/C2/C3/C5 — the star; E's router, E's line) +import routes_usage # noqa: E402 (wave 35 R9/C7 — the AI usage meter; E's router, E's line) +import routes_feedback # noqa: E402 (wave 35 R8/C6 — feedback to the operator plane; E's router) +import routes_agent_harness # noqa: E402 (wave 36 R8/C4 — the agent harness store; E's router) +import routes_script_views # noqa: E402 (wave 36 R3/R10/C3 — script Views; E's router) +import routes_geo # noqa: E402 (wave 37 R14/A11 — the keyless map provider seam; D's router) +from core import grid_events # noqa: E402 +# D-315 / D-305 (2026-08-18) — the two store REFUSALS get their own app-level handlers below. +# ⚠ Neither is a `StoreUnavailable` subclass, on purpose: routes that degrade a store outage into a +# session fallback must NOT degrade a refusal, or the user is told a change was saved that was not. +from core import data_binding # noqa: E402 +from core import store # noqa: E402 +from deps import Session, module_gate # noqa: E402 + +_WEB_DIST = _HERE.parents[1] / "web" / "dist" + +# --- THE CANONICAL FIELD CONTRACT: a startup assertion, and the referee gate's subject ---------- +# Every field tags its semantic TYPE and its SOURCE. `source='odoo'` is READ-ONLY (Odoo is never +# written); `source='overlay'` is the editable stratum that lives OUTSIDE Odoo. Loaded from the +# ONE canonical file shared with the embedded host (`platform/aios_grid_fields.json`), so +# embed == standalone by construction. +# +# TWO REASONS THIS IS HERE and not folded into the routes: +# 1. It FAILS LOUDLY at import when the file is missing — which means the deploy or the RI_DIR +# layout is broken, and discovering that from a 500 on the first customer request instead of +# at startup costs a debugging session. (The pre-wave main.py had this guard; the EXIT-2a +# rewrite dropped it and this restores it.) +# 2. `aios-web/verify_fields_contract.py` — the cross-side REFEREE — reads `FIELDS` and +# `_PASSTHROUGH_KEYS` from this module to prove the canonical file, `aios_grid.py` and this +# API have not drifted. The rewrite removed them and the referee went red; retargeting the +# gate would have been the wrong repair ([[gate-can-report-green-on-nothing]]: retarget, do +# not delete — but only when the subject genuinely moved. Here it should not have moved). +# +# ⚠ THE ROUTES SERVE A SUPERSET OF THIS. `routes_customers._payload` derives its field list from +# `aios_grid.fields_from_workspace(ws)`, which is `FIELDS` PLUS the session user's own custom_ and +# measure_ columns — the same list the Streamlit host renders. That is the point: the standalone +# shell now sees the user's own columns instead of the bare base contract. `FIELDS` is the +# canonical FLOOR, asserted below to be exactly what `aios_grid` starts from. +import json as _json_contract # noqa: E402 +_FIELDS_PATH = _RI / "aios_grid_fields.json" +if not _FIELDS_PATH.is_file(): + raise FileNotFoundError( + f"AIOS web API: canonical field contract missing at {_FIELDS_PATH}. Set RI_DIR to the " + "platform root (it also carries the data layer this API imports)." + ) +_contract = _json_contract.loads(_FIELDS_PATH.read_text(encoding="utf-8")) +FIELDS = _contract["fields"] if isinstance(_contract, dict) else _contract +# text/status/select/date pass through untouched; every OTHER odoo field is numeric -> rounded. +# Derived from field TYPE (not a hand-kept key list) so a new text/date field can never be +# wrongly rounded. `select` joined 2026-08-02 (dba) — a choice label rounded would be garbage. +_PASSTHROUGH_KEYS = {f["key"] for f in FIELDS + if f["source"] == "odoo" and f["type"] in ("text", "status", "select", + "date")} + +app = FastAPI(title="AIOS web API") + +# The customers payload measured 1.15 MB of JSON on the live Space, shipped UNCOMPRESSED — with +# the 754 KB bundle behind it, most of "the app is slow" was bytes on the wire. gzip takes the +# payload to ~10–15% of that. minimum_size spares the tiny acks the overhead. +app.add_middleware(GZipMiddleware, minimum_size=1024) + + +@app.exception_handler(StarletteHTTPException) +def _error_shape(request: Request, exc: StarletteHTTPException): + """ONE error shape for every non-2xx (X2): `{"error": {"code", "message"}}`. + + `deps.err` already raises detail in that shape; anything FastAPI raises on its own (a 404, a + 422 from a malformed path param) is wrapped here so a client never has to branch on two + different error bodies. + """ + detail = exc.detail + if isinstance(detail, dict) and "error" in detail: + body = detail + else: + body = {"error": {"code": f"http_{exc.status_code}", "message": str(detail)}} + return JSONResponse(body, status_code=exc.status_code, + headers=getattr(exc, "headers", None)) + + +@app.exception_handler(grid_events.StoreUnavailable) +def _store_unavailable(request: Request, exc: grid_events.StoreUnavailable): + """THE SAFETY NET for "the store is down" → 503, from anywhere. + + ⚠ WHY THIS IS APP-LEVEL AND NOT A `try` PER ROUTE. It was a try per route first, and a + `StoreUnavailable` raised while BUILDING THE PAYLOAD — before the route reached its own + try/except — surfaced as a 500. A store outage is a normal operational state and every route + here touches the store at least twice (the workspace read, then the write), so "remember to + wrap it" is a rule that gets forgotten once and then reports the wrong thing. One handler + means a store outage can only ever be a 503, whichever call raised it. + + The seam raises this only when the caller passed no `fallback_ws` — i.e. exactly on this + adapter, which has no durable session dict to degrade into. A 200 over a write that + evaporated is the failure the whole rule exists to prevent. + """ + return JSONResponse( + {"error": {"code": "store_unavailable", + "message": "the tenant store is unavailable. No change was saved"}}, + status_code=503) + + +@app.exception_handler(data_binding.StoreWriteRefused) +def _store_write_refused(request: Request, exc: data_binding.StoreWriteRefused): + """D-315 — this deployment may not write that store. 503, with the REASON. + + ⛔ IT ECHOES THE EXCEPTION'S OWN MESSAGE, WHERE ITS NEIGHBOUR ABOVE USES A FIXED SENTENCE, and + the difference is the point. "The store is unavailable" is the whole truth about an outage; a + refusal has a cause the operator cannot otherwise discover, because per D-160 the Space + environment cannot be read from outside. A refusal that said only "unavailable" would send + somebody hunting for a network fault that does not exist ([[report-the-cause-before-you-fix-it]]). + The message names the deployment, the store and the fix, and carries no customer data. + + ⛔ AND IT IS ITS OWN HANDLER RATHER THAN A SUBCLASS OF `StoreUnavailable`. Several routes carry + `except StoreUnavailable:` blocks that DEGRADE to a session-scoped fallback workspace — correct + for an outage, catastrophic here, because the user would be told the change was saved. A + distinct type falls through every one of them to this handler. + """ + return JSONResponse( + {"error": {"code": "store_write_refused", + "message": f"this deployment may not write the tenant store. No change was " + f"saved. {exc}"}}, + status_code=503) + + +@app.exception_handler(store.StoreConflict) +def _store_conflict(request: Request, exc: store.StoreConflict): + """D-305 — the document moved under this write more times than it could be rebased. 503. + + ⚠ 503 RATHER THAN 409, DELIBERATELY. A 409 invites the client to resolve a conflict, and there + is nothing here for it to resolve: the store already re-read and re-applied this change up to + `_MAX_REBASE` times before giving up. What the caller needs to know is that the write did not + land and nothing was damaged, which is the same contract as the outage above and the same + retry-later shape. ⛔ The one answer this must never be is 200: reaching this handler means the + only way to have "succeeded" was to overwrite somebody else's work. + """ + return JSONResponse( + {"error": {"code": "store_conflict", + "message": f"somebody else changed this database while your change was being " + f"saved, and it could not be merged. Nothing was saved or " + f"overwritten. Try again. {exc}"}}, + status_code=503) + + +@app.get("/api/health") +def health(): + """Unauthenticated LIVENESS only — it must answer before anyone can sign in, so it may not + reveal anything about the deployment beyond "the process is up". No version, no tenant list, + no config: a health endpoint is the one URL every scanner finds first. + + ⛔ THE VERSION DOES NOT GO HERE, and it was asked to (2026-08-04, when LIVE became a pinned + release and "what is LIVE running" needed an answer). A build identifier tells an unauthenticated + caller exactly which commit's known issues apply. It rides `GET /api/v1/settings` instead, behind + a session — and the authoritative copy is the `VERSION` file in the Space repo, which + `deploy_web.space_version()` reads without needing the app to be up at all.""" + return {"ok": True} + + +app.include_router(routes_auth.router) +app.include_router(routes_nav.router) +app.include_router(routes_customers.router) +app.include_router(routes_products.router) +app.include_router(routes_assets.router) +app.include_router(routes_tables.router) +app.include_router(routes_grid.router) +app.include_router(routes_records.router) +# EXIT wave 2: the Y1 page-data envelope (one route for every ported dashboard) and Y4's user +# administration. `routes_pages` imports `pages`, which lazily imports each `pages_*` builder — so +# a new page is a builder module plus one registry line, and nothing here changes. +app.include_router(routes_pages.router) +app.include_router(routes_admin.router) +app.include_router(routes_automation.router) +app.include_router(routes_keychain.router) +app.include_router(routes_statements.router) +# Wave 19 (owner item 13, R3): the LOOPABLE admin plane — the platform's own cross-tenant view. +# Its own prefix (`/api/v1/platform-admin`), never nested under `/admin`, so there is no path +# ambiguity with `routes_admin`'s `{username}` params and no chance of a tenant-admin route and a +# platform-operator route ever shadowing each other. Every path it declares is gated by +# `core.platform_admin.is_platform_admin`, and `verify_api` enumerates this router to prove it. +app.include_router(routes_platform_admin.router) +# Wave 20 (owner items 25 + 18/23/26): the Alerts inbox and the manage-access surface. Both are +# session-gated rather than admin-gated — an alert is a person's own subscription, and sharing is +# something every user does with their own views/folders/databases. +app.include_router(routes_alerts.router) +app.include_router(routes_shares.router) +app.include_router(routes_uploads.router) +# ⭐ WAVE 23 — THE THREE NEW ROUTERS. Mounted here, above the static catch-all at the bottom of this +# file, because `app.mount("/", _AppStatic(...), html=True)` swallows everything it is reached by: +# a router included AFTER it answers 404 forever while importing fine, type-checking fine and +# passing its own gate. That is the wave-20 declared-but-unmounted shape with a different cause. +# +# ⛔ ALL THREE EXISTED, COMPLETE AND GATED, WITH NO MOUNT until the close-out audit — three finished +# features that would have shipped dead. The workers each posted a mount ask and said they would +# signal "ready" first; C waited for a signal that never came while the files landed anyway. **The +# lesson is not "read the mailbox harder": `verify_api`'s enumeration is what caught it, so the +# control is that the enumeration must NAME every router in this file.** +app.include_router(routes_templates.router) # item 7 / C12 — template registry, session-gated +app.include_router(routes_connectors.router) # item 10 / C11 — the connectors directory +# ⚠ routes_forms is the FIRST NEW PUBLIC DOOR since the `_DEV_FIXTURES` scar (see :257 below). Its +# two paths are DELIBERATELY unauthenticated — a form is filled in by someone with no account — so +# it joins `/api/health`, the automation hook and the tick on the exempt list. It resolves a token +# by scanning tenants with a constant-time compare and answers a uniform 403, so a bad token cannot +# distinguish "no such form" from "not yours", and it never echoes a tenant, table or slug. +app.include_router(routes_forms.router) # item 8 / C9 — the PUBLIC form door +# WAVE 27 item 17 — E's Odoo relational doors. Mounted in the SAME change that E's module landed, +# because the wave-23 scar is exactly this: three finished routers shipped with no include_router +# line — complete, gated, type-clean and 404 for every caller. `verify_api.section_mounts` walks +# `main.app.routes` and pins these two paths by NAME, so an unmounted router now goes RED. +app.include_router(routes_odoo_tables.router) +# ⭐⭐ WAVE 31 · T49 / C4 — THE SOURCE-NEUTRAL DOOR TO THE SAME CAPABILITY. +# `/api/v1/connected-tables/{key}/rows` is an ALIAS: every request lands in +# `routes_odoo_tables.odoo_table_rows`, so "the Odoo path behaves byte-identically" holds by +# CONSTRUCTION rather than by two implementations that agree on the day they were written. R2 puts +# Meta Ads on the same mirror, and a Meta campaign served from a URL with `odoo` in it is a name +# that lies to every network tab, log line and bug report. +# ⚠ MOUNTED IN THE SAME CHANGE AS THE MODULE, per the line above and for the same wave-23 scar. +app.include_router(routes_connected_tables.router) +# ⭐⭐ WAVE 29, item 20 / R11 / contract C6 — F's CHANGE TOKEN. `GET /api/v1/changes?scope=` +# answers "did this bucket change" for ~zero cost (an in-memory counter; ZERO `store.get()` deep +# copies), which is what lets a filtered view pick up a row created in another tab, by an automation +# or by a connector sync without re-downloading the world. +# ⛔ THIS LINE IS THE ARTIFACT THIS PROTOCOL LOSES MOST RELIABLY, AND IT WAS ALREADY LOST ONCE HERE: +# F's router and the CLIENT half both shipped complete, so the poller was calling `/api/v1/changes` +# six times a minute and taking a 404 while every one of F's own gates was green. `verify_api`'s D-48 +# leg caught it (`unmatched: [('/api/v1/changes', 'apiBridge.ts')]`) — a client fetch path with no +# mounted route — which is precisely the control the wave-23 scar above was written to install. +# ⚠ Mounted is NOT callable: `section_changes_callable` in `verify_api.py` SIGNS IN and CALLS this +# route, because a route can be mounted and still raise before its own `try:` (D-107's plain-text 500). +app.include_router(routes_changes.router) # item 20 / C6 — F's router, A's line +# ⛔ WAVE 31 (R10 / C5) — E's router, taken by the INTEGRATOR under W31-T08's own done-when +# ("no router ships unmounted") because `main.py` is D's fence and D's queue did not reach it. +# It was written, gated and 404-dead: `verify_web_agent.py` asserts THIS LINE and was red at +# 60/61 for it. Four waves of the same defect — three routers in wave 23, four features in +# wave 29 — is why the assertion exists and why the line is not left for later. +app.include_router(routes_web_agent.router) # R10 / C5 — E's router, A's line +# ⭐⭐ WAVE 32 (R1 / C5, cross-fence wiring 6) — THE QUERY MODULE. E's router, A's line, and the +# FIFTH consecutive wave in which this exact line is the artifact the protocol nearly loses. +# ⛔ MOUNTED HERE, IN THIS BLOCK, AND NOT AT THE END OF THE FILE — measured by SESSION E in its own +# gate before it front-inserted: `include_router` APPENDS, and `app.mount("/", _AppStatic(...), +# html=True)` swallows everything reached after it, so a router added below that mount answers +# **405 on POST and 404 on GET** while every one of its own tests passes. The comment at :213 states +# the rule; E's measurement is what turns it from advice into a number. +# ⚠ `verify_api` asserts `/api/v1/query` in `app.openapi()["paths"]` — NEVER `{r.path for r in +# app.routes}`, which finds nothing in this app because FastAPI wraps included routers (W31). +app.include_router(routes_query.router) # R1 / C5 — E's router, A's line +# ⭐⭐ WAVE 33 (C2) — THE SIXTH CONSECUTIVE WAVE IN WHICH THIS BLOCK IS THE THING THE PROTOCOL LOSES. +# Contract C2 is written for exactly that: a lane creating a `routes_*.py` posts an ASK, and the +# INTEGRATOR adds the line in the SAME wave. Both of these arrived that way (C's `ASK C-1`, G's +# `ASK G-1`), and `verify_api::section_w23_mounts` asserts each path in `app.openapi()["paths"]` +# with an NC that comments a mount out and goes RED. +# ⚠ Same placement rule as the line above — ABOVE the `app.mount("/", _AppStatic(...), html=True)` +# at the end of the file, never after it. +# ⭐⭐ THE PUBLISH DOOR IS MOUNTED AGAIN (2026-08-15). It was withheld for the wave-33 deploy +# because QA reproduced three HIGH defects that are armed ONLY by mounting it. All three are fixed +# in `routes_publish.py`, each at its cause rather than at its symptom: +# 1. W33-T68 — `form` LEFT `PUBLISHABLE_MODES`. A form view's rows ARE the submissions people +# sent it, so publishing one served other people's answers to anyone holding the link. A form +# still has its own public door (`#/form/`) which serves the BLANK form and never rows. +# 2. W33-T69 — `_visible_keys` returns `[]` when a view STORED a `visible` list and none of its +# keys survive, instead of falling back to the table default. The fallback still applies to a +# view that never stored one, which is what it was written for. +# 3. W33-T70 — three separate leaks closed: the rate limit no longer keys on the caller-supplied +# `x-forwarded-for` (it keys on the socket peer and counts FAILURES only, so a shared proxy +# peer cannot become one global bucket); every failing path now spends the same PBKDF2 the +# success path spends, closing the 206x timing gap; and an unknown token on the sibling GET +# answers the LOCKED shape rather than a 403, so that route stops sorting real tokens from +# fake ones for free. +# ⚠ `verify_api::section_w23_mounts` asserts this path in `app.openapi()["paths"]`, so its four +# EXPECTED reds should now go GREEN. A red here after this line means the mount broke, not the gate. +app.include_router(routes_publish.router) # R5 / C2 — C's router, A's line (the publish door) +# ⛔ NOT BEHIND `module_gate("product_data")`, WHICH IS THE WHOLE REASON IT IS A SECOND ASSET DOOR. +# `routes_assets.py` gates EVERY one of its routes on that module, so a connector logo served from +# there would 403 for any account without the product-data grant — i.e. the Connectors directory +# would lose its logos for exactly the accounts most likely to be setting a connector up (C6). +app.include_router(routes_brand.router) # R4 / C2 / C6 — G's router, A's line (brand marks) +# ⛔ TWO OF ITS SIX PATHS ARE UNAUTHENTICATED BY DESIGN (`/slack/events`, `/slack/interact`) and +# that is the DOOR, not an omission — Slack posts to them with no session and could not carry one. +# They are built on `routes_forms`' proven public shape: signing-secret verification, a sliding-window +# rate limit, a body cap and ONE non-oracle 403, so a bad signature cannot distinguish "no such +# workspace" from "not yours". ⚠ There is no auth middleware and no exempt-path allow-list in this +# file to register them in: "public" here IS the absence of `Depends(require_session)`, which is why +# `verify_api` asserts the absence rather than an entry in a list that does not exist. +app.include_router(routes_slack.router) # R4 / C2 — D's router, A's line (Manage agent + Slack) +# ⭐⭐ WAVE 35 (R4/R5/R7, contracts C2/C3/C5) — THE STAR. Mounted in the SAME change that created +# `routes_starred.py`, which is the seventh consecutive wave in which this block is the artefact the +# protocol nearly loses — and the first in which the router's own lane also owns `main.py`, so there +# is no ask/mount pair to drop. `W35-T46` asserts every one of this wave's paths in +# `app.openapi()["paths"]` and CALLS one route from each. +# ⚠ Same placement rule as every line above: ABOVE `app.mount("/", _AppStatic(...), html=True)`, or +# the router answers 404 on GET and 405 on POST while every one of its own tests passes. +app.include_router(routes_starred.router) # R4 / C2 — the star, counts, and record stars +# ⛔ R9 SAYS THIS ROUTE REPORTS AND NEVER ENFORCES, so mounting it cannot cut anybody off — there is +# no ceiling anywhere behind it (`usage_ledger` has no refusal in it). The one AI limit the product +# enforces is per COLUMN and is unchanged (`ai_enrich.ceiling_report`). +app.include_router(routes_usage.router) # R9 / C7 — GET /usage, the one AI meter +# ⛔ ITS TWO DOORS CARRY DIFFERENT WALLS ON PURPOSE (R8, and D-221 is the booked precedent): the +# POST is any authenticated session's own act, the GET is `is_platform_admin` only. Mounting it does +# not widen anything a tenant admin can reach — `verify_api` proves that by having one try. +app.include_router(routes_feedback.router) # R8 / C6 — feedback to the operator plane +# ⭐⭐ WAVE 36 (R8 / C4) — THE AGENT HARNESS FILE STORE, mounted in the SAME change that created +# `routes_agent_harness.py`. Eighth consecutive wave in which this block is the artefact the +# protocol nearly loses; `verify_web_agent::section_w36_harness` asserts this path in +# `app.openapi()["paths"]` and CALLS the route, because mounted is not callable (D-107). +# ⚠ Placement, as for every line above: ABOVE `app.mount("/", _AppStatic(...), html=True)` at the +# end of this file, or a GET answers 404 and a PUT answers 405 while every gate stays green. +# ⛔ ITS PATHS SIT UNDER `/agents/{id}/...`, WHICH `routes_slack` ALSO SERVES — and that is safe +# rather than lucky: a FastAPI path parameter never spans a `/`, so `/agents/{agent_id}` cannot +# match `/agents/x/harness`. The two routers share a prefix and no route. +app.include_router(routes_agent_harness.router) # R8 / C4 — versioned agent harness files +# ⭐⭐ WAVE 36 (R3 / R10 / C3) — THE SCRIPT VIEW, owner item 6. Mounted in the SAME change that +# created `routes_script_views.py`; `verify_script_views.py` asserts both of its paths in +# `app.openapi()["paths"]` AND calls them, and its NC comments this line out. +# ⛔ ITS RUN DOOR SPAWNS A SUBPROCESS AND IS A PLAIN `def`, so FastAPI runs it in the threadpool. +# Mounting it does not put a ten-second wait anywhere near the event loop; the router's own header +# says why that is not a style choice. +app.include_router(routes_script_views.router) # R3 / R10 / C3 — code-script database Views +# ⭐⭐ WAVE 37 (R14, amendment A11) — THE KEYLESS MAP PROVIDER SEAM, mounted on `ASK D-1` in the +# same wave that created `routes_geo.py`. NINTH consecutive wave in which this block is the +# artefact the protocol nearly loses, and the first in which the ask that carries it was itself +# lost: D raised it inside a ``` fence, so `wave_board.py` blanked it and the bus never delivered +# it (D-366). The integrator found it reading the peer mailbox by hand. +# ⚠ Placement, as for every line above: ABOVE `app.mount("/", _AppStatic(...), html=True)` at the +# end of this file, or GET answers 404 and POST answers 405 while every one of its own gates is +# green. D is adding a `web_map` leg that reads THIS FILE and asserts this line — keep it. +# ⛔ R14 put the provider behind a seam precisely so swapping Google in later is a config change: +# tiles = OpenStreetMap · routing = OSRM · geocoding = Nominatim, none of them keyed, all of them +# rate-limited by POLITENESS rather than by a bill (~1 req/s, and bulk harvesting is forbidden). +app.include_router(routes_geo.router) # R14 / A11 — D's router, A's line (map providers) + + +# --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working) +# They are the v1 handlers with the v1 session requirement — NOT the old unauthenticated Basic +# behavior. An alias that kept the old auth would be a bypass of everything above it. +_GATE = module_gate(routes_customers.MODULE) + + +@app.get("/api/customers", deprecated=True) +def _customers_alias(session: Session = Depends(_GATE)): + return routes_customers._payload(session) + + +@app.patch("/api/customers/{pid}", deprecated=True) +def _patch_alias(pid: int, body: dict = Body(default=None), + session: Session = Depends(_GATE)): + return routes_customers.patch_customer(pid, body, session) + + +def _assert_contract_floor(): + """The canonical FIELDS must be exactly what `aios_grid` starts an empty workspace from. + + This is what stops `FIELDS` becoming a constant that exists only to satisfy a gate. If the + canonical JSON and `aios_grid.fields_from_workspace({})` ever disagree, the standalone API and + the embedded host are serving two different schemas and the grid's own contract has forked — + the exact drift `verify_fields_contract.py` was written to catch, now also caught at startup + on whatever machine is actually running. + """ + import aios_grid + base = [f for f in aios_grid.fields_from_workspace({}) if not f.get("custom")] + if [f["key"] for f in base] != [f["key"] for f in FIELDS]: + raise RuntimeError( + "AIOS web API: the canonical field contract and aios_grid.fields_from_workspace({}) " + "disagree on the base field set. Embed and standalone would serve different schemas. " + "Run aios-web/verify_fields_contract.py.") + + +def _startup_notes(): + """Say the two things an operator must know, once, at import — never in a response body.""" + if aios_session.EPHEMERAL_SECRET: + print("[aios-api] AIOS_SESSION_SECRET is not set. Signing with a random per-process " + "key. Sessions will not survive a restart and will not work across workers. " + "Set it in production.") + if os.environ.get("AIOS_INSECURE_SSL") == "1": + print("[aios-api] AIOS_INSECURE_SSL=1. TLS verification is DISABLED for outbound " + "requests. Local development only; never in a deployed environment.") + + +_assert_contract_floor() +_startup_notes() + + +# ⛔ THE STATIC MOUNT IS NOW UNAUTHENTICATED, and that is a change this wave made on purpose. +# Before EXIT-3a, `BasicAuth` middleware gated the WHOLE app including this mount. A branded login +# page cannot live behind a password prompt, so the shell's own assets must be public — which they +# are: `index.html`, the JS/CSS bundle and the favicon reveal nothing. +# +# ⚠ WHAT THAT SILENTLY DE-GATED, caught in review rather than in production. `web/dist/` also +# carries `sample_customers.json`, a DEV FIXTURE copied from `web/public/`. Its 8 customers are +# synthetic, but the `agent` column holds REAL EMPLOYEE NAMES, and it went from Basic-gated to +# publicly fetchable in this commit. Nothing needs it: both the API bridge and `useCustomerData` +# deleted their sample fallback on purpose ("NOTHING HERE FALLS BACK TO sample_customers.json"), +# and it survives only because it sits in `web/public/`. So it is refused here — 404, the same +# answer as any other path that is not part of the app. +# +# The right long-term fix is deleting it from `web/public/` (S2's lane — flagged in the mailbox); +# this guard is what makes the API safe regardless of what the bundle happens to contain. +_DEV_FIXTURES = {"sample_customers.json"} + + +class _AppStatic(StaticFiles): + async def get_response(self, path, scope): + if Path(path).name in _DEV_FIXTURES: + raise StarletteHTTPException(status_code=404, detail="Not Found") + resp = await super().get_response(path, scope) + # Vite content-hashes everything under assets/ (a change is a NEW url), so those are + # immutable — a repeat visit re-downloads zero bytes instead of the whole 750 KB bundle. + # index.html must stay revalidated or a deploy would strand returning browsers on the old + # bundle; ETag/304 makes that revalidation a header exchange, not a transfer. + # ⚠ Normalised first: on a Windows host StaticFiles hands this path with backslashes, + # and `startswith("assets/")` silently skipped every asset (measured on the local probe). + if path.replace("\\", "/").lstrip("/").startswith("assets/"): + resp.headers["Cache-Control"] = "public, max-age=31536000, immutable" + else: + resp.headers["Cache-Control"] = "no-cache" + return resp + + +# static bundle LAST so /api/* wins; html=True serves index.html at / plus the built assets +if _WEB_DIST.is_dir(): + app.mount("/", _AppStatic(directory=str(_WEB_DIST), html=True), name="web") + + +# --- boot prewarm (AIOS_PREWARM=1 — set by the Dockerfile, never by tests) ----------------------- +# Without this, the first visitor after every deploy/restart pays the full Odoo pool build in +# their request. The thread warms the CONSOLIDATED scope (None, None) + every registered page's +# default envelope; scoped users still pay their own scope's first build, once. +# Env-gated rather than a startup event so importing `api.main` in a gate (verify_api and friends +# run against fakes) can never fire a live Odoo pull. +def _seed_and_sync_store(): + """Bring the analytical store (harness.datastore) LIVE for this container — the app.py + bootstrap, mirrored (2026-07-31, owner item 1). + + `harness.datastore` powers every measure column/condition, and ONLY app.py used to call + `ensure_seed()` — so on a fresh Space disk this container resolved measures against a store + that never existed and every measure cell served blank. Seeding alone was NOT enough either, + and that was measured live the same day: "datastore seeded in 1.7s" followed by an endless + `api:measure-column: ModelError the data cache is still warming up` — `ready()` demands + EVERY entity at phase 'live', and a seed that predates a newer entity leaves it un-synced + forever in a process with no sync loop. The SYNC SPRINT after the seed is what closes the + write_date gap and backfills anything the seed lacks (app.py:7779's exact pattern, bounded + passes). Fail-quiet throughout: no seed/token/Odoo → the columns stay blank, the rows still + serve. + """ + import time as _t + t0 = _t.time() + try: + from harness import datastore as _ds + if _ds.ensure_seed(): + print(f"[aios-api] datastore seeded in {_t.time() - t0:.1f}s") + # DEDICATED Odoo connection for this thread (the W6 postmortem rule): the sync's + # search_reads must never interleave on the shared client's xmlrpc transport. + import core.odoo as _odoo + try: + _odoo._tlocal.client = _odoo.OdooClient() + except Exception: + pass + res = {} + for i in range(12): + res = _ds.sync_all(log=lambda *a, **k: None) + print(f"[aios-api] datastore sync pass {i + 1}: " + + ", ".join(f"{k}={v.get('phase')}" for k, v in sorted(res.items()))) + if all(v.get("phase") == "live" for v in res.values()): + break + # ⭐ Wave 21 (item 2, "make sure the metrics are correct"): a cursor sync can never see + # a HARD DELETE, and the downloaded seed carries whatever was deleted since it was cut — + # one reconcile pass at boot removes both classes of phantom row before the first + # measure is served. MEASURED 2026-08-05: five deleted sale_order_line rows = $284.25 of + # phantom YTD revenue, stable across re-syncs, zero the moment reconcile ran. + try: + _ds.reconcile_deletes(log=lambda *a, **k: None) + except Exception: + pass + print(f"[aios-api] datastore sync done in {_t.time() - t0:.1f}s " + f"(ready={_ds.ready()})") + # ⭐⭐ 2026-08-09 (wave 28, D-107) — THE RELATIONAL REBUILD RUNS AT BOOT, HERE. + # + # ⛔ IT DID NOT BEFORE, AND NOTHING SAID SO. The rebuild lived only inside + # `_store_resync_loop`, whose very first statement is `sleep(1800)` — so the earliest a + # freshly booted container could spawn the four locked databases was T+30 MINUTES. The + # symptom was read as "the boot path is silent": `/odoo-tables/status` polled every 30 s + # across a ~20-minute window over two boots returned the pre-wave schema on all 20 + # samples. It was not silent, it had not been asked yet. Both halves of D-107 were like + # this — a thing that never ran, mistaken for a thing that ran and failed. + # + # ⚠ WHY HERE AND NOT IN `_prewarm`: the tables are DERIVED FROM THE MIRROR, and this is + # the exact line where the mirror has finished advancing — seed, up to twelve sync passes, + # then the delete reconcile. Calling it from the other thread would race the seed and hit + # either `ro_con()`'s "still warming" RuntimeError or, worse, a HALF-SYNCED mirror, which + # is D-107's own hypothesis 3: a partial population trips `MAX_SHRINK` and the rebuild + # refuses — correctly, but for a reason that reads like a data loss scare. + # ⚠ Same thread on purpose: it is already a daemon and nothing serves requests behind it. + # ⭐⭐ W35-T45 / R11 — TENANT #0'S ODOO CREDENTIAL MOVES ONTO ITS KEYCHAIN, IN THE CONTAINER. + # ⚠ BEFORE the relational rebuild, deliberately: the rebuild resolves its connector through + # `rt.odoo_source()`, so running the migration first means the very next read already goes + # through the keychain branch and a broken migration is visible in THIS boot's log rather + # than in tomorrow's. It is idempotent, so every later boot is one `list_entries` read. + _migrate_env_odoo("boot") + _pull_meta("boot") + _rebuild_odoo_relational("boot") + # ⭐⭐ W32-T07 — owner items 13 and 15, DELIVERED. See `_sweep_automation_schemas`. + # ⚠ AFTER the two above and not before: those advance the mirror and can take minutes, and + # this sweep is unrelated to it — putting it last means a slow Odoo sync cannot delay the + # one thing on this path that fixes a grid the owner has asked about twice. + _sweep_automation_schemas("boot") + except Exception as e: # noqa: BLE001 + print(f"[aios-api] datastore seed/sync skipped: {e}") + + +def _migrate_env_odoo(why): + """⭐⭐ W35-T45 / R11 — move tenant #0's environment Odoo credential onto its keychain. + + ⛔⛔ IN THE CONTAINER, WHICH IS THE WHOLE REASON THIS IS A BOOT LINE AND NOT A SCRIPT. D-195, + measured three times: a developer's CLI write to the tenant store is reverted by the running + Space within a minute (download-modify-upload, last-write-wins) — and the write REPORTS SUCCESS + every time, then a fresh read confirms it, and it is gone by the next poll. A CLI migration would + be a dry run that lies, and what it would lie about here is a credential. + + ⚠ IDEMPOTENT AND SCOPED TO TENANT #0 by `routes_keychain.env_odoo_available`, so on every other + tenant and on every later boot this is one `list_entries` read and a line. + ⚠ FAIL-QUIET: this must never take a boot down. But it is never SILENT — a skip prints its reason, + because "already migrated", "no keychain key on this deployment" and "the env is incomplete" are + three different operator actions and a blank Keychain page cannot tell them apart. + """ + try: + import routes_keychain as _kc_routes + from harness import runtime as _runtime + rep = _kc_routes.migrate_env_odoo(_runtime.get_runtime("royal-imports")) + if rep.get("done"): + print(f"[aios-api] odoo credential migrated onto the keychain ({why}): " + f"entry={rep['entry']} carried_pause={rep['carried_pause']}" + + (f" PROBLEM: {rep['why']}" if rep.get("why") else "")) + else: + print(f"[aios-api] odoo keychain migration skipped ({why}): {rep.get('why')}") + except Exception as e: # noqa: BLE001 + print(f"[aios-api] odoo keychain migration FAILED ({why}): {type(e).__name__}: {e}") + + +def _pull_meta(why): + """Pull Meta Ads into THIS container's mirror, before the relational rebuild reads it. + + ⛔ WHY IT HAS TO HAPPEN HERE AND NOT ON A LAPTOP. The mirror is a FILE that lives beside the + process; the Space's copy is seeded from the HF dataset and knows nothing about a DuckDB on a + developer's box. Running `meta_store --sync` locally populates the local mirror and the LIVE + product stays empty — which is the whole difference between "the loader works" and "the + product has the data". Odoo is already arranged this way (`sync_all` runs in the container); + this is the same arrangement for the second connector. + + ⚠ FAIL-QUIET AND SILENT WHEN THERE IS NOTHING TO DO. No token => no Meta => one line, no + error: a tenant that has not connected Meta is a normal state, and this runs on every boot. + ⚠ The window is deliberately SHORT here (`META_INSIGHTS_DAYS`, default 7 at boot) because boot + is not the place for a 90-day backfill — the resync pass widens it. + """ + try: + from harness import meta_store as _meta + if not _meta.token(): + print(f"[aios-api] meta sync skipped ({why}): no META_ADS_ACCESS_TOKEN in this " + f"deployment - the connector is idle, not broken") + return + # ⛔ PASSED, NOT SET IN THE ENVIRONMENT. `meta_store.INSIGHTS_DAYS` binds at + # IMPORT, so an `os.environ.setdefault` here executed after the module was + # already loaded and changed NOTHING: every boot pulled 90 days instead of 7, + # which is the slow path that trips the per-ad-account rate limit and never + # finishes. A knob read at import cannot be turned by a caller at runtime. + rep = _meta.sync("royal-imports", log=lambda *_a: None, insights_days=7) + for p in rep.get("problems") or []: + print(f"[aios-api] meta sync PROBLEM ({why}): {p}") + print(f"[aios-api] meta sync done ({why}): " + + ", ".join(f"{k}={v['in_mirror']}" for k, v in sorted(rep["tables"].items()))) + except Exception as e: # noqa: BLE001 + print(f"[aios-api] meta sync FAILED ({why}): {type(e).__name__}: {e}") + + +def _rebuild_meta_relational(why, rt): + """Spawn/refresh the `ut_meta_*` locked databases off the SAME mirror the Odoo half just used. + + ⭐ CALLED FROM INSIDE `_rebuild_odoo_relational`, ON PURPOSE, and the reason is D-29 rather than + tidiness: `harness/datastore` is a ONE-FILE-AT-A-TIME process global, and that caller has just + established which tenant's file this process holds open. Spawning Meta here inherits that + binding instead of rebinding it under live readers — which is the documented way to serve one + tenant's rows to another with nothing raised. + + ⚠ SILENT WHEN THERE IS NOTHING TO DO. A tenant that never connected Meta has no `meta_*` tables + in its mirror; `refresh` returns `{}` and says so once. That is a normal state, not a failure, + and it must not print an error every 30 minutes for every tenant that does not use Meta. + ⛔ Its own try/except for the reason the two passes above have theirs: a Meta refusal must not + cancel an Odoo rebuild that already succeeded. + """ + try: + import meta_relational as _meta + counts = _meta.refresh(rt, why) + if counts: + print(f"[aios-api] meta relational rebuild done ({why}): " + + ", ".join(f"{k}={v}" for k, v in sorted(counts.items()))) + try: + import automation_engine as _engine + _engine.refresh_relations(rt, log=lambda *_a: None) + except Exception as e: # noqa: BLE001 + print(f"[aios-api] meta relation cells failed ({why}): {type(e).__name__}: {e}") + except Exception as e: # noqa: BLE001 + print(f"[aios-api] meta relational rebuild FAILED ({why}): {type(e).__name__}: {e}") + + +def _sweep_automation_schemas(why): + """⛔⛔ WAVE 32 · `W32-T07` — MAKE D'S DECLARATIONS REACH TENANTS THAT ALREADY HAVE THE TABLES. + + Owner items 13 and 15 — the TikTok comments lock, and the comment CONTENT column he says he has + asked for *"many times"*. **Both were already correct in the source.** `TT_COMMENT_FIELDS` + carries `field_def("text", "Comment")` and `TT_LOCKED_TABLES` already contains the comments + table, both since wave 31, with his words quoted in the comment beside them. So this ticket is + not a schema change and there is nothing to design: it is DELIVERY. + + ⛔ THE MECHANISM, WHICH IS THE WHOLE OF IT. `ut_ensure` MERGES fields into an existing table and + stamps `recordMode` — but only **when something calls it**, and the only callers are automation + runs. A tenant whose `ut_tt_comments` was spawned before the declaration changed keeps the old + shape until an automation happens to run against it. Nothing sweeps existing tenants. That is + [[a-migration-that-runs-on-the-next-write]], and it is this wave's stated thesis: a declaration + that never reaches a tenant is indistinguishable from one that was never written. + + ⛔⛔ AND IT MUST RUN **IN THE CONTAINER**, WHICH IS WHY THIS IS IN `main.py` AND NOT A SCRIPT. + D-195, measured three times: a developer's CLI write to the tenant store is reverted by the + running Space within a minute (download-modify-upload, last-write-wins) — and **the write + reports success every time**, then a FRESH read confirms it, and it is gone by the next poll. + A connector's tables must be spawned BY THE CONTAINER; a CLI spawn is a dry run that lies. + + ⚠ EVERY TENANT, unlike `_rebuild_odoo_relational` below — and the asymmetry is deliberate + rather than an oversight. That function is scoped to royal because it derives from the DuckDB + mirror, and `harness/datastore` is a ONE-FILE-AT-A-TIME process global (D-29): rebinding it per + tenant in a daemon thread can serve one tenant's rows to another with nothing raised. This + sweep touches only `user_tables` through each tenant's own `rt`, which has no such global — so + the hazard that scopes that one does not exist here, and TikTok automations run in tenants + other than #0. + + ⚠ CHEAP ON A CORRECT TENANT: `ut_ensure` short-circuits when nothing changed, so this is a read + per child table on a tenant that is already right, and the whole delivery on one that is not. + ⚠ ONE TENANT'S FAILURE MUST NOT STOP THE NEXT. Each is wrapped: a tenant whose store is + unreachable at boot is reported and skipped, never allowed to abort the sweep for everyone. + """ + try: + import automation_engine as _eng + from harness import runtime as _runtime + except Exception as e: # noqa: BLE001 + print(f"[aios-api] schema sweep ({why}) SKIPPED. Import failed: {e}") + return + try: + schemas = _eng.platform_schemas() + except Exception as e: # noqa: BLE001 + print(f"[aios-api] schema sweep ({why}) SKIPPED. No declarations: {e}") + return + tenants = [] + try: + tenants = _runtime.known_tenants() + except Exception as e: # noqa: BLE001 + print(f"[aios-api] schema sweep ({why}) SKIPPED. Tenant list unreadable: {e}") + return + for slug in tenants: + try: + rt = _runtime.get_runtime(slug) + # ⛔⛔ FIXED 2026-08-13 — THIS SWEEP WAS SPAWNING EIGHT DATABASES IN EVERY TENANT. + # Owner: *"Database for Royal Imports, why we have fucking IG and TIktok databases."* + # `ut_ensure`'s first line is *"Create the table if it is missing"*, and this loop fed + # it every child of every platform schema for every tenant — so tenant #0, a floral and + # giftware importer with ZERO automations, woke up on 2026-08-13 at 12:05 UTC owning + # `ut_ig_posts`, `ut_ig_comments`, `ut_ig_snapshots`, `ut_ig_post_snapshots` and the + # four TikTok twins, all empty, all in his nav flyout. **The bug is one word wide:** + # this function's own title says *"MAKE D'S DECLARATIONS REACH TENANTS THAT ALREADY HAVE + # THE TABLES"* and its body called a CREATE-OR-MERGE function to do a MERGE-ONLY job. + # [[reuse-and-delete-are-hypotheses]] — `ut_ensure` was the right function for the + # merge and brought a second behaviour nobody wanted with it. + # + # ⭐ THE PREDICATE IS "DOES THIS TENANT ALREADY HAVE THE TABLE", read ONCE per tenant + # rather than per child — `rt.get` deep-copies the whole tenant document (28.6 MB on + # tenant #0), so asking eight times is eight copies to answer one question. + # ⚠ AND IT MUST NOT WEAKEN THE DELIVERY: a tenant that HAS `ut_tt_comments` still gets + # the `text` column and the `recordMode` stamp, which is the entire point of T07. The + # sweep now delivers to tables that exist and mints none, which is what it always + # claimed to do. + have = set(rt.get("user_tables") or {}) + ensured, skipped = [], [] + for s in schemas: + for key, child in (s.get("children") or {}).items(): + if key not in have: + skipped.append(key) + continue + got = _eng.ut_ensure(rt, child["label"], child["fields"], "automation", + key=key, lock_fields=True, + record_mode=child["record_mode"]) + if got: + ensured.append(got) + if skipped: + # ⭐ SAID OUT LOUD, never silently skipped — this repo's "no silent caps" rule. A + # sweep that quietly does nothing looks identical to a sweep that is not running, + # which is how the previous behaviour survived review in the first place. + print(f"[aios-api] schema sweep ({why}) {slug}: {len(skipped)} child table(s) not " + f"present in this tenant, so nothing was created for them " + f"({', '.join(sorted(skipped))}). They are spawned by an automation that " + f"needs them, never by this sweep") + # ⭐ THE RETRACTION (D's `retract_foreign_presets`, D-152), AFTER the children loop. + # ⛔ THE SWEEP ABOVE MAKES COLUMNS **ARRIVE** AND CANNOT MAKE STALE ONES **LEAVE**, and + # T07's `done-when` asserts both ("no `ut_tt_*` grid carries an Instagram column"). On a + # tenant that ran TikTok before W30-T08 the 26 machine-authored IG columns are still + # there — the detector was fixed, the damage never was. + # ⚠ `kept` IS THE HONEST HALF: a foreign column that HOLDS DATA is REPORTED, never + # deleted. If it is non-empty, the screenshot shows a column and the report is the + # answer (W30/R6's second sentence). + st = {} + try: + st = _eng.retract_foreign_presets(rt, log=lambda *a, **k: None) or {} + except Exception as e: # noqa: BLE001 + print(f"[aios-api] schema sweep ({why}) {slug}: retraction FAILED: {e}") + print(f"[aios-api] schema sweep ({why}) {slug}: ensured={len(ensured)} " + f"tables={st.get('tables', 0)} columns={st.get('columns', 0)} " + f"cells={st.get('cells', 0)} flags={st.get('flags', 0)} " + f"kept={st.get('kept') or []}") + except Exception as e: # noqa: BLE001 + print(f"[aios-api] schema sweep ({why}) {slug}: FAILED: {e}") + # ⭐ A SUCCESS MARKER, for `_rebuild_odoo_relational`'s stated reason: D-107 was chased for a + # day on ABSENT log markers, which cannot tell "it ran and was fine" from "it was never + # reached". Three failure markers and no success marker makes silence ambiguous. + print(f"[aios-api] schema sweep ({why}) done over {len(tenants)} tenant(s)") + + +def _rebuild_odoo_relational(why): + """Spawn/refresh the four locked Odoo databases, then compute their cells. `why` is 'boot' or + 'resync' and rides every log line, because "it failed" and "it failed at boot, before anyone + could have asked" are different diagnoses. + + ⭐ THE SUCCESS LINE IS NOT DECORATION — it is the control this path lacked. D-107 was chased + for a day on the strength of *absent* log markers, which cannot distinguish "the rebuild ran + and was fine" from "the rebuild was never reached". Three failure markers and no success + marker means silence is ambiguous; now it is not. + + ⚠ SCOPED TO ROYAL-IMPORTS, deliberately, and it is NOT the D-29 shortcut it resembles. The + caller has just advanced whichever DuckDB file this process holds open — tenant #0's — and + royal is the only tenant with an Odoo mirror to derive from (R1). Iterating tenants here walks + straight into D-29's documented hazard: `harness/datastore` is a ONE-FILE-AT-A-TIME + process-global, and rebinding it in a daemon thread under live readers can serve one tenant's + rows to another with nothing raised. `is_royal` stays the authority on which slugs qualify. + """ + try: + import odoo_relational as _rel + from harness import runtime as _runtime + if not _rel.is_royal("royal-imports"): + return + _rt = _runtime.get_runtime("royal-imports") + counts = _rel.refresh(_rt, "royal-imports") + # ⭐⭐ AND THEN COMPUTE THE CELLS, which `refresh` does NOT do. + # + # ⛔ THE FAILURE THIS CLOSES IS THE WORST-LOOKING KIND. `refresh` writes rows and field + # DEFINITIONS; every Link and Rollup cell comes from a separate pass. Those passes used to + # live only on the automation `tick`, which fires from an EXTERNAL EventBridge cron — so a + # fresh boot landed 71,954 rows with eleven fully-configured relational columns and every + # one of them BLANK until an unrelated scheduler happened to run. Nothing errors; the + # tables simply look finished and answer nothing. + # + # ⚠ `refresh_relations`, NOT `compute_relation_cells`. The latter computes a change count + # over a blob it was handed and PERSISTS NOTHING; the former walks the tenant and writes. + # Calling the inner one here would return a plausible number and change no cell. + # + # ⚠ ONE try/except PER PASS, for the reason `tick` states at its own copies: a source + # rollup REFUSES loudly on a truncated group set, and a shared block would let that honest + # refusal silently cancel a relational pass that had already succeeded. + try: + import automation_engine as _engine + _engine.refresh_relations(_rt, log=lambda *_a: None) + except Exception as e: # noqa: BLE001 + print(f"[aios-api] odoo relation cells failed ({why}): {type(e).__name__}: {e}") + try: + import rollup_sql as _rollup + for _b, _key, _l, _f in _rel.TABLES: + _rollup.compute(_rt, _key) + except Exception as e: # noqa: BLE001 + print(f"[aios-api] odoo source rollups failed ({why}): {type(e).__name__}: {e}") + print(f"[aios-api] odoo relational rebuild done ({why}): " + + ", ".join(f"{k}={v}" for k, v in sorted((counts or {}).items()))) + _rebuild_meta_relational(why, _rt) + except Exception as e: # noqa: BLE001 + # ⚠ The TYPE is named here as it is in the two inner handlers. The original printed only + # `{e}`, and a bare message is exactly what made D-107 unreadable from the outside: a + # `BinderException` about a missing column is a different action from a timeout or an auth + # failure, and the text alone often does not say which it was. + print(f"[aios-api] odoo relational rebuild failed ({why}): {type(e).__name__}: {e}") + + +def _store_resync_loop(): + """Keep the analytical mirror current — the API-process stand-in for the re-sync the + Streamlit app piggybacks on page renders. Measure memos key on the pool stamp, so a + refreshed pool re-reads the freshly synced store.""" + import time as _t + passes = 0 + while True: + # ⭐⭐ WAVE 32 · OWNER ITEM 11 / R11 — THE TENANT'S OWN CADENCE, NOT A HARDCODED 1800. + # + # ⛔ THIS LINE IS WHY THE SETTING WAS NOT A SETTING. SESSION B built the whole of item 11 — + # the 30m/1h/4h/daily/manual presets, the server-side clamp, the config door and + # `odoo_relational.sync_seconds()` which turns the stored preset into seconds — and its own + # ticket said the one-line change belonged to A because `main.py` is A's file. That ASK was + # never sent, so the value was stored, displayed, clamped and IGNORED: a person could pick + # "every 4 hours" and the loop would keep resyncing every 30 minutes with nothing anywhere + # reporting the disagreement. Found by `verify_reachability` naming `sync_seconds` as a + # function whose ONLY caller was its own gate [[artifact-with-no-importer]]. + # + # ⚠ `None` MEANS MANUAL AND MUST NOT MEAN ZERO. R11's `manual` preset returns None from + # `sync_seconds`; treating that as a falsy interval would spin this loop with no sleep at + # all. It parks at the default cadence instead and simply does no work — the tenant asked + # not to be synced automatically, not for the server to stop breathing. + # ⚠ RE-READ EVERY PASS, deliberately: a cadence changed in Settings takes effect on the + # next cycle rather than at the next container restart, which is what makes it a setting. + # ⚠ FAIL-SAFE TO 1800 — an unreadable config must not become a tight loop. The floor is + # enforced in `sync_seconds` as well as at the write door, for the same reason. + # ⛔⛔ WAVE 34 · W34-T46 / D-246 — THE PARAGRAPH ABOVE DESCRIBED A BRANCH THAT DID NOT + # EXIST, AND THAT IS WHY NOBODY LOOKED. It said `manual` "parks at the default cadence + # instead and simply DOES NO WORK". The first half was true; the second half was not + # implemented anywhere: `_secs is None` set the sleep to 1800 and then fell into an + # UNCONDITIONAL `sync_all()`. So a tenant who chose "manual" was synced every thirty + # minutes exactly like everyone else, with the setting stored, displayed, clamped and + # obeyed by nothing — the identical failure this block's own W32/R11 note is about, one + # layer further in. A comment asserting a fix is why the second reader stops reading. + # ⚠ FIXED, AND THE FIX IS NARROW ON PURPOSE. `manual` is an ODOO connector setting, so it + # skips the Odoo mirror sync and its reconcile and NOTHING ELSE: the Meta pull and the + # relational rebuild below still run, because a tenant asking not to have Odoo polled has + # not asked for the other connector to stop. + # ⚠ AND IT STILL SLEEPS AND STILL LOOPS. The config is re-read every pass, so switching + # back off `manual` takes effect on the next cycle rather than at the next restart. + _every = 1800 + _manual = False + try: + import odoo_relational as _rel_cad + from harness import runtime as _rt_cad + # ⚠ D-245, KNOWN AND NOT FIXED HERE: the slug is hardcoded, so this reads TENANT #0's + # cadence and applies it to a loop that syncs the shared mirror. Correct while one + # tenant has Odoo; wrong the moment a second one does. Out of this ticket's scope and + # named rather than silently inherited. + _secs = _rel_cad.sync_seconds(_rt_cad.get_runtime("royal-imports")) + _manual = _secs is None + _every = 1800 if _secs is None else max(int(_secs), 60) + except Exception: # noqa: BLE001 + pass + _t.sleep(_every) + passes += 1 + try: + from harness import datastore as _ds + import core.odoo as _odoo + try: + _odoo._tlocal.client = _odoo.OdooClient() + except Exception: + pass + # ⚠ A PLAIN BRANCH, NOT AN EARLY EXIT AND NOT AN EXCEPTION. A `raise` here would be + # caught by this block's own `except` and printed as "store resync failed", turning a + # setting a person chose into a recurring error in the log; a `continue` would skip + # the Meta pull and the relational rebuild below, which `manual` says nothing about. + if not _manual: + _ds.sync_all(log=lambda *a, **k: None) + # Wave 21 — every 4th pass (~2h): purge hard-deleted rows the cursor sync cannot + # see (the boot pass's comment has the measured case). Cheap id-sweep per entity; + # without it deleted Odoo lines inflate every sum on the mirror FOREVER. + # ⚠ INSIDE the branch: a reconcile is a pass over the mirror this loop was just + # told not to advance. + if passes % 4 == 0: + _ds.reconcile_deletes(log=lambda *a, **k: None) + except Exception as e: # noqa: BLE001 + print(f"[aios-api] store resync failed: {e}") + # ⭐ WAVE 27 item 17 (E's ASK ->A, resolved): the Odoo RELATIONAL tables are rebuilt + # AFTER the mirror they are derived from, in the same pass and in that order — deriving + # from a mirror this loop is about to advance would publish a worklist one cycle stale + # every single time. + # + # ⛔ OUTSIDE the try/except above, NOT folded into it. A failing relational rebuild must + # not swallow the sync's error message, and — worse the other way — a sync failure must + # not skip a rebuild that had nothing wrong with it. Two independent failures, two + # independent logs. (`_rebuild_odoo_relational` carries its own handlers.) + # + # ⚠ THE REBUILD IS NOT THE FRESHNESS GUARANTEE — the rows carry a visible `_refreshed` + # stamp for that. This loop is a daemon thread whose failure path is a `print` (D-29), so + # "the wiring exists" and "the data is current" are different claims and only the stamp + # can tell a user which one they are looking at. + # + # ⭐ ONE IMPLEMENTATION, TWO CALLERS (wave 28). This block used to be the only copy, which + # is what made the boot path silent for 30 minutes after every restart; it is now the + # SECOND caller of the same function `_seed_and_sync_store` calls at boot. A copy here + # would be a second thing to keep in step, and the two would answer differently on the + # next ruling — the exact shape the Views top-up was just fixed for on the other side of + # this wave. + # ⛔ THE META PULL RIDES THE RESYNC TOO, AND LEAVING IT OUT WAS A REAL GAP — caught by + # reading the deploy's own boot log rather than by any gate. `_pull_meta` was wired into + # `_seed_and_sync_store` ALONE, i.e. it ran exactly once per container, at boot, behind a + # full Odoo sync. So a boot where the Graph call was rate-limited, slow or simply after the + # thread died left the mirror empty with NOTHING to retry it: the relational rebuild below + # would then find no `meta_*` tables every 30 minutes forever and skip, silently and + # correctly. A connector that can only ever be established at boot is one bad boot away + # from being permanently absent. + # ⚠ Cheap when there is nothing to do: no token => one line and return. + _pull_meta("resync") + _rebuild_odoo_relational("resync") + + +def _prewarm(): + import time as _t + t0 = _t.time() + # ⚠ The store sync runs in its OWN thread, never ahead of the pool warm: a stale seed can + # take many minutes of XML-RPC to close, and the first live probe of this arrangement + # showed the pool warm (42s) silently queued behind it — the whole app cold for every + # visitor while a background column channel caught up. Measure cells retry via the + # transient rule until the sync lands; nothing else waits on it. + import threading as _th + _th.Thread(target=_seed_and_sync_store, daemon=True, name="store-seed-sync").start() + try: + from harness import runtime as _runtime + rt = _runtime.get_runtime("royal-imports") + routes_customers.warm_default(rt) + import pages as _pages + _pages.warm_default(rt) + # ⭐⭐ WAVE 30 · T12, THE COLD PATH (owner items 4/5). Automation was the ONE module absent + # from this list, so its memo was always filled by a visitor rather than by boot: call 1 of + # `GET /automations` after every deploy downloaded the whole `user_tables` document (35.8 MB + # ceiling, under the store lock) onto whoever clicked first. Memoising the WARM path — two + # waves of it — could not touch that, because the cold call is the one that fills the memo. + # ⚠ It elects a STRING and keeps no document; see `warm_default`'s own note on why caching + # the bucket would trade a latency for memory this tier does not have. + import routes_automation as _rauto + _rauto.warm_default(rt) + print(f"[aios-api] prewarm done in {_t.time() - t0:.1f}s") + except Exception as e: # noqa: BLE001 — boot must not die on a warm-up + print(f"[aios-api] prewarm skipped: {e}") + + +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ W31-T46 / D-160 — THE MIRROR IS SEEDED WHETHER OR NOT `AIOS_PREWARM` IS SET. +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# +# THE DEFECT, and it has cost a release once already (owner item 12, wave 20). `_prewarm()` was +# the ONLY caller of `_seed_and_sync_store()`, which is the ONLY caller of +# `datastore.ensure_seed()`. So on a fresh Space disk with `AIOS_PREWARM` anything but `1`: +# no `royal.duckdb` ⇒ `datastore.ready()` False forever ⇒ `ro_con()` refuses ⇒ every measure +# column blank AND — since wave 30 put the Odoo grids on the mirror — **two user-facing grids +# serve nothing**, with the app RUNNING, the deploy green and the tag correct. Last time the +# symptom was read as a pinned tag and a store problem for days. +# +# ⛔ AND THE FLAG IS EASIER TO LOSE THAN IT LOOKS: `deploy_web.py` pushes `AIOS_PREWARM=1` +# EXPLICITLY (to overwrite a stale `0`, because a Space secret survives a redeploy) — but that +# push sits under `if TARGET:`, so an ordinary bare `python deploy_web.py` skips it. A guard +# written for the dangerous case that the ordinary case walks straight past. +# +# ⭐ SO THE SEED IS SPLIT OFF AND MADE UNCONDITIONAL, which is T46's first branch rather than its +# fallback. It is the right half to move because it is the CHEAP, SAFE one: `ensure_seed()` +# touches no Odoo (it is one `hf_hub_download` of a snapshot), returns immediately when the file +# is already there, and returns immediately without `HF_TOKEN`. The EXPENSIVE, live half — +# `sync_all()`'s XML-RPC passes, the pool warm, the resync loop — stays exactly where it was, +# because the env gate's stated reason is still true: importing `api.main` in a gate must never +# fire a live Odoo pull. +# +# ⚠ THE `DB_PATH.exists()` PRE-CHECK IS WHAT KEEPS THIS FREE. It is a stat, and it is False only +# on a genuinely fresh disk — so on every developer box and in every gate run the thread is never +# started at all, and on a fresh Space it does exactly the thing whose absence blanks the grids. +#: What the boot seed did, so a surface can REPORT it instead of an operator inferring it from a +#: blank grid. R6's second sentence: a limit that cannot be removed is reported with its cause. +MIRROR_SEED = {"attempted": False, "seeded": False, "cause": "", "recommendation": ""} + + +def _seed_mirror_if_absent(): + """Hydrate the analytical mirror when this container has none — INDEPENDENT of `AIOS_PREWARM`. + + Fail-quiet by design, and it records WHY rather than only whether: "there is no mirror and no + HF_TOKEN to fetch one" and "there is no mirror and the fetch failed" are different operator + actions, and a blank grid cannot tell them apart. + """ + from harness import datastore as _ds + MIRROR_SEED["attempted"] = True + try: + if _ds.ensure_seed(): + MIRROR_SEED["seeded"] = True + print("[aios-api] analytical mirror seeded at boot (independent of AIOS_PREWARM)") + return True + if not _ds.DB_PATH.exists(): + MIRROR_SEED["cause"] = ( + "this container has no analytical mirror and the seed snapshot could not be " + "fetched (no HF_TOKEN, or the dataset was unreachable)") + MIRROR_SEED["recommendation"] = ( + "set HF_TOKEN on the deployment; until then every connected grid and every " + "measure column served from the mirror is empty") + print(f"[aios-api] NO ANALYTICAL MIRROR: {MIRROR_SEED['cause']}") + except Exception as e: # noqa: BLE001 — boot must not die + MIRROR_SEED["cause"] = f"the boot seed raised {type(e).__name__}: {e}" + MIRROR_SEED["recommendation"] = "check HF_TOKEN and the seed dataset's availability" + print(f"[aios-api] mirror seed skipped: {e}") + return False + + +try: + from harness import datastore as _ds_boot + if not _ds_boot.DB_PATH.exists(): + import threading as _threading_seed + _threading_seed.Thread(target=_seed_mirror_if_absent, daemon=True, + name="mirror-seed").start() +except Exception as e: # noqa: BLE001 + print(f"[aios-api] mirror seed not scheduled: {e}") + +if os.environ.get("AIOS_PREWARM") == "1": + import threading as _threading + _threading.Thread(target=_prewarm, daemon=True, name="prewarm").start() + _threading.Thread(target=_store_resync_loop, daemon=True, name="store-resync").start() diff --git a/api/odoo_relational.py b/api/odoo_relational.py index 48eebac0c9c2b04f53e0769a863ec03560421c2b..d4c2a9d0c709c7e50b54f78709d0f29d138c24a2 100644 --- a/api/odoo_relational.py +++ b/api/odoo_relational.py @@ -1,1849 +1,1849 @@ -"""odoo_relational.py — Odoo entities as LOCKED relational databases. - -Owner ruling R1 / contract C8: spawn preset Odoo databases for Royal Imports, give them preset -Link + Rollup fields, and prove each rollup against the `measure_` column it will eventually -replace. - -⭐⭐ 2026-08-09 — THE POPULATIONS WIDENED FROM "OPEN AR" TO **EVERY ODOO ID**, which is the -owner's item: *"make sure we have all the Unique ID in Odoo in Database for the Royal Imports -tenant."* Before this change there were two tables holding 438 partners and 1,228 invoices — the -partners who owed money — so most Odoo ids were simply absent, and the missing rows were the -reason a Rollup could not answer a sales question. Four tables now, keyed on the Odoo id itself: - - ut_odoo_customers 2,465 rows 0.43 MB every partner with a confirmed order or a - posted customer document - ut_odoo_products 5,829 rows 1.20 MB every active product carrying a SKU code - ut_odoo_invoices 31,418 rows 8.68 MB EVERY posted customer invoice + refund - ut_odoo_orders 32,700 rows 7.50 MB every confirmed sale order - -⛔ WHAT MADE THAT LEGAL, AND IT WAS NOT A BIGGER NUMBER. `MAX_ROWS` was 5,000 and this module's -own `plan()` refused above it — but the cap was never a property of the store (see the measured -banner on `core.user_tables.MAX_ROWS`; `ig_master` has run a 500,000-row bucket the whole time). -The cap is now 60,000, DERIVED from what a row actually weighs (⚠ this line said 100,000 until -wave 28 — that was the FIRST candidate and its own derivation REJECTED it for clearing the memory -budget by 0.6%; the prose was written before the number lost, and two sibling files said it too). -The four tables together are -17.81 MB in one `user_tables` document — real, bounded, and booked: the per-table row-key split -is D-87's next increment. ⛔ ORDER LINES REMAIN OUT (256,810 rows / 63.9 MB / 2.57 s per copy); -they are answered by the read-through rollup, which never copies a row. - -⭐ THE EXCLUDED CHANNEL IS NOW A COLUMN, NOT A DELETION. `core.odoo.EXCLUDE_PARTNER_NAMES` puts -GIFTWARE DEALS (partner 6369 — the Amazon channel) outside WHOLESALE scope, and the old tables -dropped its rows entirely. Dropping them contradicts "every Odoo id", so the rows are kept and -carry **`wholesale_scope`** instead. ⚠⚠ READ THIS BEFORE COMPARING ANY TOTAL: that one partner -holds **$1,755,779.95 of the $2,347,608.49** raw open balance — 75% of it — across 25 invoices. -Wholesale open AR is $591,828.54. So a column total here will not equal the AR page unless you -filter `wholesale_scope`, and that is the scope difference, not a defect. `read_open_ar` keeps -excluding, because `modules/ar` is its oracle and an oracle answers ONE question. - -⭐ AR SURVIVED THE WIDENING UNCHANGED, AND THAT IS MEASURED, NOT ASSUMED. `sum(residual)` over -ALL posted customer documents equals `sum(residual)` over `modules/ar._open_docs`' own predicate -**to the cent** ($2,347,608.49): zero posted rows carry a non-zero residual outside -`payment_state IN ('not_paid','partial')`, and zero rows inside it carry a residual of 0. So the -`ar_outstanding` rollup needs no condition. ⛔ THE COUNT AND THE DATE DO — `countall` over the -wider link would count 31,418 documents and call them open invoices, so those two rollups carry -the oracle's predicate as an explicit `payment_state` condition pair. -""" -import datetime as _dt - -#: Royal Imports only (R1). A tenant slug that is not this one gets a refusal, never a spawn: -#: nurilab has no Odoo mirror behind these tables and would get empty locked databases. -RI_SLUGS = ("", "royal-imports") - -INVOICES_KEY = "ut_odoo_invoices" -CUSTOMERS_KEY = "ut_odoo_customers" -ORDERS_KEY = "ut_odoo_orders" -PRODUCTS_KEY = "ut_odoo_products" -#: ⭐ WAVE 28 (owner R1): *"ALL of Unique ID in Odoo is a database e.g. Customers/Products/Agents, -#: etc. Including expenses and GL codes."* Four more DOCUMENT/REGISTRY grains, each measured to -#: fit far inside `MAX_ROWS` (19 / 192 / 6,538 / 393 against 60,000). -AGENTS_KEY = "ut_odoo_agents" -ACCOUNTS_KEY = "ut_odoo_accounts" -BILLS_KEY = "ut_odoo_bills" -VENDORS_KEY = "ut_odoo_vendors" - -#: ⭐⭐ WAVE 30 / R7 / W30-T35 — THE TWO LINE GRAINS, AND THEY ARRIVE THE ONLY WAY THEY EVER COULD. -#: -#: ⚠ THE PARAGRAPH THAT STOOD HERE SAID THESE WERE "DELIBERATELY NOT HERE … at any cap", and it -#: was RIGHT ABOUT THE CAP AND WRONG ABOUT THE CONCLUSION — which is exactly why it is replaced -#: rather than left standing beside its own contradiction. The obstacle was never the number of -#: rows; it was that every row had to be COPIED into the shared `user_tables` document. MEASURED -#: on this box's mirror 2026-08-12: 254,189 order lines in the confirmed scope (256,810 unscoped) -#: and 963,783 GL lines — 4.2x and 16x `MAX_ROWS`, 63.9 MB and ~240 MB as JSON. Owner ruling R6 -#: settles what that means: *"there is no cap in how many data from the API source … can be pulled -#: into the app"*, so the answer is a different residency, never a bigger ceiling. -#: -#: ⛔ THESE TWO TABLES STORE NO ROWS HERE AND NEVER WILL. `routes_odoo_tables` binds them to the -#: DuckDB mirror (`GRID_SOURCES`) and `core.user_tables.row_limit` answers **0** for them — "this -#: database stores no rows HERE", which is a different statement from `None` ("connected and -#: uncapped") and from `MAX_ROWS` ("the editable substrate"). `plan()` below reads that evaluator -#: and builds no python row for either grain: 963,783 dicts in one process is the dangerous work -#: the answer exists to prevent. What DOES get written is the DEFINITION — a locked database with -#: fields, a label, grants and a nav entry, and zero rows. A definition with no rows is a working -#: grid; that is the whole shape of the conversion. -ORDER_LINES_KEY = "ut_odoo_order_lines" -GL_LINES_KEY = "ut_odoo_gl_lines" - -#: The join column the partner-grain tables carry. Derived links resolve through it (`on`/`from`). -JOIN_KEY = "partner_id" -#: The product-grain equivalent. -PRODUCT_JOIN_KEY = "product_id" -#: ⚠ AN AGENT IS A `res.partner`, so its id shares the partner namespace with a customer's — but -#: it is a DIFFERENT COLUMN on the customer row (`agent_id`, the customer's assigned agent) and the -#: two must never be joined through `JOIN_KEY`, which would link every customer to itself. -AGENT_JOIN_KEY = "agent_id" -#: A vendor is also a `res.partner`; same reasoning, its own column. -VENDOR_JOIN_KEY = "vendor_id" -ACCOUNT_JOIN_KEY = "account_code" - -#: The oracle's own predicate — `modules/ar._open_docs`, copied rather than re-derived so the two -#: cannot drift. It now selects a SUBSET of the invoices table rather than defining it. -_AR_OPEN = "payment_state IN ('not_paid','partial')" -_POSTED_DOCS = "state = 'posted' AND move_type IN ('out_invoice','out_refund')" -_AR_WHERE = f"{_POSTED_DOCS} AND {_AR_OPEN}" -_CONFIRMED = "state IN ('sale','done')" - -#: A refresh that would delete more than this share of a table's stored rows REFUSES instead. -#: ⛔ THE GUARD ONLY BECAME NECESSARY WHEN THE TABLES GOT BIG. `_ensure_table` removes rows that -#: left the population, which is right — a reversed invoice must not keep inflating a total. But -#: the population comes from the DuckDB mirror, and a mirror caught mid-resync (or one seeded -#: against an empty store) answers with FEWER rows and no error. At 1,228 rows that was a visible -#: mistake; at 31,418 it is a silent one. Odoo history does not halve, so a halving is a bad read. -MAX_SHRINK = 0.5 - - -def _ut(): - import core.user_tables as user_tables - return user_tables - - -def _registry(): - """`core.registry`, imported the same lazy way `_ut` is — this module is imported by the - route layer before `platform/` is necessarily on the path.""" - import core.registry as registry - return registry - - -def _iso_today(): - return _dt.date.today().strftime("%Y-%m-%d") - - -def _preset(field, flow="odoo_relational"): - """Stamp a field as machine-owned + preset — the `ut_ensure` lock_fields convention, so the - grid renders it grey and the preset walls refuse a rename or a delete.""" - field = dict(field) - field["automation"] = {"flowId": flow, "preset": True} - return field - - -#: The two conditions that reproduce `modules/ar`'s open-document predicate inside a rollup. -#: ⚠ Two `eq` legs joined by OR, not one `in` — `ROLLUP_CONDITION_OPS` has no `in`, and inventing -#: one here would be a second condition vocabulary beside `_clean_rollup`'s. -_OPEN_ONLY = {"conditions": [{"field": "payment_state", "op": "eq", "value": "not_paid"}, - {"field": "payment_state", "op": "eq", "value": "partial"}], - "conditionConj": "or"} - - -# --------------------------------------------------------------------------------------------- -# FIELD CONTRACTS -# --------------------------------------------------------------------------------------------- -# ⚠ Every type here must be in `core.user_tables.UT_FIELD_TYPES`, and `_clean_field` returns None -# for an unknown one — which DELETES the column silently on the next read rather than erroring. -def _scope_field(): - return {"key": "wholesale_scope", "label": "In wholesale scope", "type": "checkbox", - "source": "overlay", "default": False, - "description": "Unticked = the GIFTWARE DEALS / Amazon channel, which every wholesale " - "metric in this product excludes. The row is kept so no Odoo id is " - "missing; filter on this column to reconcile against the AR page."} - - -def _refreshed_field(): - return {"key": "refreshed", "label": "Refreshed", "type": "date", "source": "overlay", - "default": False, "description": "When this row was last reconciled against Odoo."} - - -def agent_fields(): - """One row per SALES AGENT, keyed on the `res.partner` id. - - ⭐ THE POPULATION IS A UNION OF TWO DISAGREEING SOURCES, and the disagreement is the reason it - is a union rather than a pick. MEASURED 2026-08-09: 16 partners carry commission lines, 17 - carry `res_partner.agent = TRUE`, and the union is 19 — so **2 agents earn commission without - the flag and 3 are flagged with no commission yet**. Either source alone silently drops real - agents. Same shape as `read_customers`' two document universes, for the same reason. - """ - return [_preset(f) for f in ( - {"key": "agent", "label": "Agent", "type": "text", "source": "overlay", - "default": True, "pinned": True}, - {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `res.partner` id. Also this row's id."}, - {"key": AGENT_JOIN_KEY, "label": "Odoo agent id", "type": "int", "source": "overlay", - "default": False}, - {"key": "flagged", "label": "Flagged in Odoo", "type": "checkbox", "source": "overlay", - "default": True, - "description": "Ticked = `res.partner.agent` is set. Unticked agents were found by " - "their commission lines instead - both are real, which is why this " - "table is the union of the two."}, - {"key": "commissioned", "label": "Has commission lines", "type": "checkbox", - "source": "overlay", "default": True}, - # ⭐ THE INVERSE HALF: the customers whose `agent_id` names this agent. MEASURED: 2,093 - # customers carry one and ALL 2,093 resolve to a row in this table (zero dangling). - # ⛔ W33-T43 / AMENDMENT A2 — the `customers` REVERSE link DELETED. See `invoice_fields` - # for the ruling. ⚠ This one costs more than the other two and the difference is worth - # recording: those were a click-through to a record whose id stays on the row, while this - # was an agent's BOOK — the list of customers assigned to them. The id side survives - # (`agent_id` here, and `agent_id` on `customer_data`), so the relationship is intact in - # the data and only the rendered list is gone; the same question is answerable on the - # customer grid by filtering `agent_id`, and at analytical grain via the `agent` dim on - # `sales_lines` / `sales_orders`. - _refreshed_field(), - )] - - -def account_fields(): - """One row per `account.account` — the GL chart, the owner's "GL codes". - - ⚠ NO LINK COLUMN, and that is a finding rather than an omission. A GL account meets the rest - of this schema only at LINE grain (963,783 `account_move_line` rows, 154,917 of them on - expense-type accounts), and a `ut_*` link folds rows that live in the store. The honest - binding is a read-through rollup naming a governed topic, or the mirror grid (R2) — never a - link into a table that does not exist. Declaring one here would render a permanently blank - column, which is the exact trap D-87 warns about from the value site. - """ - return [_preset(f) for f in ( - {"key": "account_code", "label": "Code", "type": "text", "source": "overlay", - "default": True, "pinned": True}, - {"key": "account_name", "label": "Account", "type": "text", "source": "overlay", - "default": True}, - {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `account.account` id. Also this row's id."}, - # ⚠ 15 DISTINCT VALUES MEASURED IN THE MIRROR, all declared. A `select` storing a value its - # options omit is wave-26 item 24: the filter panel answers with a list that cannot match - # what is stored. - {"key": "account_type", "label": "Type", "type": "select", "source": "overlay", - "default": True, - "options": ["expense", "expense_direct_cost", "expense_depreciation", "income", - "income_other", "asset_cash", "asset_current", "asset_receivable", - "asset_fixed", "asset_non_current", "asset_prepayments", - "liability_current", "liability_payable", "liability_credit_card", - "liability_non_current", "equity", "equity_unaffected", "off_balance"]}, - {"key": "is_expense", "label": "Expense account", "type": "checkbox", "source": "overlay", - "default": True, - "description": "Ticked for the expense family - the same predicate the semantic layer's " - "gl_lines topic uses, so this column and that topic cannot disagree."}, - _refreshed_field(), - )] - - -def vendor_fields(): - """One row per partner we have POSTED a vendor bill to, keyed on the `res.partner` id. - - ⚠ A VENDOR IS NOT A CUSTOMER TABLE ROW, even though both are `res.partner`. MEASURED: 393 - vendors, of which only 9 also appear in the customer population. Pointing bills at - `ut_odoo_customers` would have dangled 384 of 393 links — the failure would have been a mostly - empty column, not an error. - """ - return [_preset(f) for f in ( - {"key": "vendor", "label": "Vendor", "type": "text", "source": "overlay", - "default": True, "pinned": True}, - {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `res.partner` id. Also this row's id."}, - {"key": VENDOR_JOIN_KEY, "label": "Odoo vendor id", "type": "int", "source": "overlay", - "default": False}, - # ⭐⭐ W33-T48 / owner item 13 — the columns the census proved are there and were not shown. - # ⚠ `default` is deliberately split: the ones a vendor list is READ for (contact + tax id) - # arrive visible; the postal lines arrive HIDDEN, because four address columns turned on by - # default would push the useful ones off the first screen, and DESIGN.md's "never - # over-explain" applies to columns as much as to prose. Every one is one click away in the - # field picker, and a locked database still allows fields (item 4's vocabulary). - {"key": "country", "label": "Country", "type": "text", "source": "overlay", - "default": True}, - {"key": "email", "label": "Email", "type": "text", "source": "overlay", "default": True}, - {"key": "phone", "label": "Phone", "type": "text", "source": "overlay", "default": True}, - {"key": "mobile", "label": "Mobile", "type": "text", "source": "overlay", - "default": False}, - {"key": "vat", "label": "Tax ID", "type": "text", "source": "overlay", "default": True, - "description": "Odoo `vat` — the vendor's tax/VAT registration number."}, - # ⚠ `vendor_ref`, not `ref`: `ref` is Odoo's own name for it, and the bills grid already - # uses `ref` for the VENDOR'S INVOICE NUMBER on a document. Two different facts, and a - # shared spelling across two linked grids is how a rollup ends up summing the wrong column. - {"key": "vendor_ref", "label": "Vendor reference", "type": "text", "source": "overlay", - "default": False, - "description": "Odoo `res.partner.ref` — our internal reference for this vendor."}, - {"key": "website", "label": "Website", "type": "url", "source": "overlay", - "default": False}, - {"key": "street", "label": "Street", "type": "text", "source": "overlay", - "default": False}, - {"key": "street2", "label": "Street 2", "type": "text", "source": "overlay", - "default": False}, - {"key": "city", "label": "City", "type": "text", "source": "overlay", "default": False}, - {"key": "zip", "label": "ZIP", "type": "text", "source": "overlay", "default": False}, - {"key": "bills", "label": "Bills", "type": "link", "source": "overlay", "default": True, - "link": {"table": BILLS_KEY, "on": VENDOR_JOIN_KEY, "from": VENDOR_JOIN_KEY}}, - _refreshed_field(), - )] - - -def bill_fields(): - """One row per POSTED vendor bill or refund — the owner's "expenses", at DOCUMENT grain. - - ⚠ DOCUMENT GRAIN IS A CHOICE AND IT IS THE ONLY ONE THAT FITS: 6,538 bills against 154,917 - expense GL lines. What a person calls "expenses" is both, and they are different tables - the - bill is what you pay, the line is what it was coded to. This is the payable; the line ledger - is the read-through mirror grid (R2). - """ - return [_preset(f) for f in ( - {"key": "bill_no", "label": "Bill", "type": "text", "source": "overlay", - "default": True, "pinned": True}, - {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `account.move` id. Also this row's id."}, - {"key": "vendor", "label": "Vendor", "type": "text", "source": "overlay", "default": True}, - {"key": VENDOR_JOIN_KEY, "label": "Odoo vendor id", "type": "int", "source": "overlay", - "default": False}, - {"key": "invoice_date", "label": "Bill date", "type": "date", "source": "overlay", - "default": True}, - {"key": "due_date", "label": "Due date", "type": "date", "source": "overlay", - "default": True}, - # ⚠ SIGNED, like the customer side: Odoo's `_signed` fields already carry the refund's - # direction, so a refund reduces a total without anybody re-deriving a sign here. - {"key": "amount_untaxed", "label": "Billed $", "type": "currency", "source": "overlay", - "default": True, "agg": "sum"}, - {"key": "residual", "label": "Outstanding $", "type": "currency", "source": "overlay", - "default": True, "agg": "sum"}, - {"key": "payment_state", "label": "Payment state", "type": "select", "source": "overlay", - "default": True, - "options": ["not_paid", "partial", "in_payment", "paid", "reversed"]}, - {"key": "move_type", "label": "Document", "type": "select", "source": "overlay", - "default": False, "options": ["in_invoice", "in_refund"]}, - {"key": "vendor_link", "label": "Vendor record", "type": "link", "source": "overlay", - "default": False, - "link": {"table": VENDORS_KEY, "on": VENDOR_JOIN_KEY, "from": VENDOR_JOIN_KEY}}, - _refreshed_field(), - )] - - -def invoice_fields(): - """One row per POSTED customer invoice or refund — the full history, not just what is open.""" - return [_preset(f) for f in ( - {"key": "invoice_no", "label": "Invoice", "type": "text", "source": "overlay", - "default": True, "pinned": True}, - {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `account.move` id. Also this row's id."}, - {"key": "customer", "label": "Customer", "type": "text", "source": "overlay", - "default": True}, - {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay", - "default": False}, - {"key": "invoice_date", "label": "Invoice date", "type": "date", "source": "overlay", - "default": True}, - {"key": "due_date", "label": "Due date", "type": "date", "source": "overlay", - "default": True}, - {"key": "residual", "label": "Outstanding $", "type": "currency", "source": "overlay", - "default": True, "agg": "sum", - "description": "Odoo's signed residual. Exactly 0 on every settled document, which is " - "why AR rollups need no filter."}, - {"key": "amount_untaxed", "label": "Invoiced $", "type": "currency", "source": "overlay", - "default": True, "agg": "sum"}, - # ⛔ THE OPTION LIST WIDENED WITH THE POPULATION. It read ['not_paid','partial'] while the - # table held open AR only; the full posted history also carries paid / in_payment / - # reversed. A select holding a value its options do not declare is the wave-26 item-24 - # defect — the filter panel answers with a list that cannot match what is stored. - {"key": "payment_state", "label": "Payment state", "type": "select", "source": "overlay", - "default": True, - "options": ["not_paid", "partial", "in_payment", "paid", "reversed"]}, - {"key": "move_type", "label": "Document", "type": "select", "source": "overlay", - "default": False, "options": ["out_invoice", "out_refund"]}, - _scope_field(), - # ⭐ THE RECIPROCAL HALF (owner item 2, 2026-08-09). DERIVED (`on` declared), exactly like - # its twin, so the engine owns the cell and no human can edit a relation Odoo decided. - # ⛔⛔ W33-T43 / AMENDMENT A2 — `customer_link` DELETED, and the loss is stated here rather - # than left to be inferred from a green gate. - # - # It pointed at `ut_odoo_customers`, which W33-T44 retires (R2: one identity per subject). - # A2 ruled OPTION 2 — retire both twins, DROP the three link columns, keep every data - # column — so this grid loses the CLICK-THROUGH to a customer record and NOTHING else: - # `customer` (the name) and `partner_id` (the Odoo id) are plain columns on this same row, - # and `customer_data` now carries `partner_id` too, so both ends of the join still exist. - # - # ⛔ THE ALTERNATIVE WAS REFUSED IN WRITING, and the reason belongs beside the deletion: - # re-pointing this bag at `customer_data` needs `core/user_tables.py::_clean_link`'s `ut_` - # prefix test relaxed — which makes EVERY GATE GREEN while - # `automation_engine::compute_relation_cells` still resolves out of the `user_tables` - # document alone and returns `{}`. Empty cells, blank rollups, nothing red. A2 forbids - # touching that line this wave for exactly that reason. - # ⭐ DEBT D-88, closed 2026-08-09. `invoice_origin` carries the ORDER NAME an invoice was - # raised from, and the mirror did not sync it until this wave — so order->invoice was a - # two-hop join through 963,783 `account_move_line` rows and wave 27 shipped no link at all. - # ⚠ IT IS A NAME, NOT AN ID, and Odoo writes free text there (a manual invoice can hold - # anything; a merged one can hold several origins space-separated). The link resolves - # against `order_no` and finds nothing when the text is not an order name — the honest - # outcome, and the reason this is a join HINT rather than a foreign key. - {"key": "origin_order", "label": "Source order", "type": "text", "source": "overlay", - "default": False, - "description": "Odoo's `invoice_origin` - usually the order name, sometimes blank."}, - {"key": "order_link", "label": "Order record", "type": "link", "source": "overlay", - "default": False, - "link": {"table": ORDERS_KEY, "on": "order_no", "from": "origin_order"}}, - _refreshed_field(), - )] - - -def order_fields(): - """One row per CONFIRMED sale order — `state in (sale, done)`, the fixed wholesale scope.""" - return [_preset(f) for f in ( - {"key": "order_no", "label": "Order", "type": "text", "source": "overlay", - "default": True, "pinned": True}, - {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `sale.order` id. Also this row's id."}, - {"key": "customer", "label": "Customer", "type": "text", "source": "overlay", - "default": True}, - {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay", - "default": False}, - {"key": "order_date", "label": "Order date", "type": "date", "source": "overlay", - "default": True}, - {"key": "amount_untaxed", "label": "Order $", "type": "currency", "source": "overlay", - "default": True, "agg": "sum"}, - {"key": "team", "label": "Business unit", "type": "select", "source": "overlay", - "default": True, "options": ["Fisch", "Royal", "Sales", "Giftware Deals"]}, - {"key": "state", "label": "State", "type": "select", "source": "overlay", - "default": False, "options": ["sale", "done"]}, - {"key": "invoice_status", "label": "Invoice status", "type": "select", "source": "overlay", - "default": True, "options": ["invoiced", "to invoice", "upselling", "no"]}, - _scope_field(), - # ⛔ W33-T43 / AMENDMENT A2 — `customer_link` DELETED here too; see the note on the same - # column in `invoice_fields` above for the ruling and the refused alternative. This row - # keeps `customer` and `partner_id`, so only the click-through is gone. - # The reciprocal of the invoice's `order_link` (D-88): the invoices raised from THIS - # order, matched on the order's own name. - {"key": "invoices", "label": "Invoices", "type": "link", "source": "overlay", - "default": True, - "link": {"table": INVOICES_KEY, "on": "origin_order", "from": "order_no"}}, - _refreshed_field(), - )] - - -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# THE TWO READ-THROUGH GRAINS (W30-T35). Their rows are SERVED FROM THE MIRROR, never stored. -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# ⛔⛔ THE KEY SET IS HALF OF A CONTRACT AND `routes_odoo_tables.GRID_SOURCES[…]["cols"]` IS THE -# OTHER HALF. It binds a key to SQL; the label, type and order are declared HERE, once. The two -# lists must name the SAME columns, and both failure directions are silent: -# * a bound key with no declaration -> a silent NO-CELL (the row projection is strict); -# * a declared key with no binding -> an INACTIVE filter leaf, which WIDENS the result set. -# `verify_scopes.section_line_grids` compares the two sets, which is why neither side may "just -# add a column". -# -# ⛔ NO LINK COLUMN AND NO `refreshed` STAMP ON EITHER TABLE, and both absences are findings -# rather than omissions: -# * a LINK folds the rows the TARGET table STORES (`compute_relation_cells` reads the raw -# document, not the mirror), so a link at a read-through grain resolves against nothing and -# renders a permanently blank column — the trap `account_fields` names from the other end. -# The join ids ride as ordinary filterable columns instead, so the relationships are all -# still reachable by a person and by SQL. -# * `refreshed` means "when this row was last reconciled against Odoo", and it is stamped by -# `_ensure_table_inplace` onto rows it WRITES. Nothing here is ever written, so the column -# would be blank for every row forever. `section_line_grids` tolerates the key; the honest -# thing is not to declare it. - - -def order_line_fields(): - """One row per `sale.order.line` on a CONFIRMED order — the same `state in (sale, done)` - scope every wholesale metric in this product uses. - - MEASURED on the mirror 2026-08-12: **254,189 lines in scope** of 256,810 (the 2,621 excluded - sit on draft/sent/cancelled orders). Every line in scope is on a `sale` order — zero `done` — - but `done` stays in the option list because it is in the SCOPE, and a filter offering only - what happens to be stored today goes stale the first time an order is marked done. - - ⛔ THE SCOPE, THE ORDER DATE AND THE ORDER NAME ALL LIVE ACROSS A JOIN. `sale_order_line` - carries no `state` at all (12 columns, measured), so the binding is a join to `sale_order` — - which is also what makes `order_no` a readable primary cell instead of a line id. - - ⚠ `qty` IS `int`, NOT `currency`, AND THAT IS A MEASUREMENT. 3,888 of 256,810 lines carry a - FRACTIONAL quantity (0.2, 0.4, 0.5, 1.66 …) and the minimum is -1.0, so the question "does the - type truncate?" had to be answered rather than assumed: it does not. `int` and `currency` both - render through the client's `numberText`, which rounds nothing without a `format.decimals` - bag — the only difference is the `$` a `currency` column prepends. A quantity is not money, so - it takes the type that does not paint one. - """ - return [_preset(f) for f in ( - {"key": "order_no", "label": "Order", "type": "text", "source": "overlay", - "default": True, "pinned": True, - "description": "The sale order this line belongs to. Zero orders have a blank name, " - "which is why it is the primary cell rather than the line id."}, - {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `sale.order.line` id. Also this row's id."}, - {"key": "order_id", "label": "Odoo order id", "type": "int", "source": "overlay", - "default": False, - "description": "The `sale.order` id — the key `ut_odoo_orders` is keyed on."}, - {"key": "customer", "label": "Customer", "type": "text", "source": "overlay", - "default": True}, - {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay", - "default": False}, - {"key": "product", "label": "Product", "type": "text", "source": "overlay", - "default": True, - "description": "Blank on the 101 section and note lines, which carry no product."}, - {"key": PRODUCT_JOIN_KEY, "label": "Odoo product id", "type": "int", "source": "overlay", - "default": False}, - {"key": "qty", "label": "Qty", "type": "int", "source": "overlay", "default": True, - "agg": "sum", - "description": "Ordered quantity. 3,888 lines carry a fraction and some are negative " - "(returns), so nothing here is rounded."}, - {"key": "price_subtotal", "label": "Line $", "type": "currency", "source": "overlay", - "default": True, "agg": "sum"}, - {"key": "margin", "label": "Margin $", "type": "currency", "source": "overlay", - "default": False, "agg": "sum", - "description": "Odoo's own line margin. Populated on every line."}, - {"key": "purchase_price", "label": "Unit cost", "type": "currency", "source": "overlay", - "default": False, - "description": "The cost Odoo priced this line's margin against, per unit."}, - {"key": "order_date", "label": "Order date", "type": "date", "source": "overlay", - "default": True}, - {"key": "state", "label": "State", "type": "select", "source": "overlay", - "default": False, "options": ["sale", "done"]}, - _scope_field(), - )] - - -def gl_line_fields(): - """One row per `account.move.line` — the general ledger, and the owner's "expenses" at the - grain a person can actually browse. - - MEASURED 2026-08-12: **963,783 lines**, of which 944,846 posted, 18,885 cancelled and 52 - draft. ⛔ UNSCOPED ON PURPOSE — a general ledger whose draft and cancelled entries are - invisible is a ledger that cannot be reconciled, so `parent_state` rides as a COLUMN and the - reader chooses. That is the same decision the binding states from the SQL side. - - ⚠ TWO COLUMNS ARE LEGITIMATELY BLANK ON REAL ROWS, named here so neither reads as a defect: - **61,911 lines carry no partner** (journal entries that are not about a customer), and - **1,727 carry no account** at all, which is also why `account_code` — the key - `ut_odoo_accounts` is keyed on — is blank on exactly those 1,727 and the join that supplies - it is a LEFT one. - - ⚠ `move_type` IS `text`, NOT `select`, and it is the `product_fields.category` argument: - five values exist today (`out_invoice` 515,634 · `entry` 414,992 · `in_invoice` 18,901 · - `out_refund` 14,061 · `in_refund` 195) and Odoo's enum is longer than what we happen to hold. - A select whose options go stale answers a filter with a list that cannot match a stored value - (wave-26 item 24). `line_type` and `parent_state` ARE selects because their option lists were - measured COMPLETE against the whole table. - """ - return [_preset(f) for f in ( - {"key": "entry", "label": "Entry", "type": "text", "source": "overlay", - "default": True, "pinned": True, - "description": "The journal entry this line belongs to. Never blank."}, - {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `account.move.line` id. Also this row's id."}, - {"key": "move_id", "label": "Odoo entry id", "type": "int", "source": "overlay", - "default": False}, - {"key": "account", "label": "Account", "type": "text", "source": "overlay", - "default": True}, - {"key": ACCOUNT_JOIN_KEY, "label": "Account code", "type": "text", "source": "overlay", - "default": True, - "description": "The GL code, from the joined chart of accounts — the key " - "`ut_odoo_accounts` is keyed on. Blank on the 1,727 lines with no " - "account."}, - {"key": "customer", "label": "Partner", "type": "text", "source": "overlay", - "default": True, - "description": "Blank on the 61,911 lines that are not about a partner."}, - {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay", - "default": False}, - {"key": "date", "label": "Date", "type": "date", "source": "overlay", "default": True}, - {"key": "debit", "label": "Debit", "type": "currency", "source": "overlay", - "default": True, "agg": "sum"}, - {"key": "credit", "label": "Credit", "type": "currency", "source": "overlay", - "default": True, "agg": "sum"}, - {"key": "balance", "label": "Balance", "type": "currency", "source": "overlay", - "default": True, "agg": "sum", - "description": "Debit minus credit, as Odoo stores it. Sums to zero over a whole entry."}, - {"key": "line_type", "label": "Line type", "type": "select", "source": "overlay", - "default": False, - "options": ["product", "cogs", "payment_term", "line_note", "line_section"]}, - {"key": "move_type", "label": "Document type", "type": "text", "source": "overlay", - "default": False}, - {"key": "parent_state", "label": "Entry state", "type": "select", "source": "overlay", - "default": True, "options": ["draft", "posted", "cancel"]}, - _scope_field(), - )] - - -def product_fields(): - """One row per `product.product`, keyed on its id — EVERY product, archived ones included. - - ⚠ THE ROW ID IS THE PRODUCT ID, NOT THE SKU CODE, and the difference is measurable: 12 codes - map to more than one product id (re-SKU / merge history). The code is what a human reads and - the id is what `sales_lines.product` groups by, so both are columns and only the id is the - identity. - - ⛔ THE PINNED COLUMN IS THE NAME, NOT THE SKU, and that is not a style choice. 62 products - carry no `default_code` at all (UBER CHARGE, Delivery Charges, PICK UP …) while ZERO carry a - blank name — measured. Pinning `code` would give those rows a blank primary cell, which is - exactly D-80: a first column nothing populates quietly becoming the row's identity - ([[fallback-that-became-the-rule]]). - """ - return [_preset(f) for f in ( - {"key": "product", "label": "Product", "type": "text", "source": "overlay", - "default": True, "pinned": True}, - {"key": PRODUCT_JOIN_KEY, "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `product.product` id. Also this row's id, and what " - "every product rollup groups by."}, - {"key": "code", "label": "SKU", "type": "text", "source": "overlay", - "default": True, "description": "Odoo's `default_code`. Blank on the 62 charge/service " - "products that are not stocked SKUs."}, - {"key": "active", "label": "Active in Odoo", "type": "checkbox", "source": "overlay", - "default": True, - "description": "Unticked = archived. Archived products are kept because they still " - "carry sales history — two of them sold this year."}, - # ⚠ TEXT, NOT SELECT. 71 categories exist today and Odoo gains them without telling us; a - # select whose options go stale answers a filter with a list that cannot match a stored - # value (wave-26 item 24). Text filters honestly and never goes out of date. - {"key": "category", "label": "Category", "type": "text", "source": "overlay", - "default": True}, - {"key": "product_type", "label": "Type", "type": "select", "source": "overlay", - "default": False, "options": ["product", "consu", "service"]}, - {"key": "standard_price", "label": "Standard cost", "type": "currency", - "source": "overlay", "default": True}, - # ⭐ SOURCE-BACKED (read-through) — the product-grain half of "compute all of the data in - # Odoo". It names a governed TOPIC + METRIC KEY and one grouped query answers every SKU; - # `sales_lines` holds 256,810 rows that are never copied into this table. - {"key": "sales_ytd", "label": "Sales YTD", "type": "rollup", "source": "overlay", - "default": True, "agg": "sum", - "rollup": {"source": {"topic": "sales_lines", "measure": "revenue", - "groupBy": "product", "on": PRODUCT_JOIN_KEY, "window": "ytd"}}}, - {"key": "units_ytd", "label": "Units YTD", "type": "rollup", "source": "overlay", - "default": True, "agg": "sum", - "rollup": {"source": {"topic": "sales_lines", "measure": "units", - "groupBy": "product", "on": PRODUCT_JOIN_KEY, "window": "ytd"}}}, - {"key": "margin_ytd", "label": "Gross margin YTD $", "type": "rollup", "source": "overlay", - "default": True, "agg": "sum", - "rollup": {"source": {"topic": "sales_lines", "measure": "margin", - "groupBy": "product", "on": PRODUCT_JOIN_KEY, "window": "ytd"}}}, - _refreshed_field(), - )] - - -def customer_fields(): - """One row per partner Odoo has transacted with, keyed on the `res.partner` id. - - ⭐ Two DERIVED links (`on` declared), so the engine owns both cells and a human cannot edit a - relation Odoo already decided. The rollups come in two kinds on purpose: - * LINK rollups fold the rows in `ut_odoo_invoices` / `ut_odoo_orders` — they can answer - anything about a document the table holds, including a date rank; - * SOURCE rollups name a governed topic + metric and are answered by ONE grouped SQL query - over the whole mirror — they can answer a DATE-WINDOWED money question, which a link - rollup cannot, because a condition can only compare against a literal and a literal year - start is right until 1 January. - """ - return [_preset(f) for f in ( - {"key": "customer", "label": "Customer", "type": "text", "source": "overlay", - "default": True, "pinned": True}, - {"key": JOIN_KEY, "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `res.partner` id. Also this row's id."}, - # ⭐⭐ W37-T11 / D-165 — THE POSTAL ADDRESS, which this table declared nowhere and served - # never. `verify_odoo_relational._prove_customer_address` recorded the blocker in its own - # docstring on 2026-08-12 — *"`street`/`street2`/`zip` are not among [the mirror's 16 - # columns] … blocked on a `sync_all()` backfill no ticket in wave 30 owns"*. That backfill - # HAS since run (`_sync_state` carries `res_partner.cols.…,street,street2,…,zip`, done - # 2026-08-15), so the half that could not be built now can be. - # ⛔ THIS IS ALSO LANE D'S DEPENDENCY. R4 puts address→coordinates in an enrichment field, - # and a geocoder pointed at a column that does not exist enriches nothing. - # ⚠ `default: False` on the postal lines, `True` on city/state — the same split - # `vendor_fields` made and for the same reason: four address columns on by default push - # the useful ones off the first screen. A locked database still allows fields, so every - # one is a click away in the picker. - {"key": "street", "label": "Street", "type": "text", "source": "overlay", - "default": False}, - {"key": "street2", "label": "Street 2", "type": "text", "source": "overlay", - "default": False}, - {"key": "city", "label": "City", "type": "text", "source": "overlay", "default": True}, - {"key": "state", "label": "State", "type": "text", "source": "overlay", "default": True}, - {"key": "zip", "label": "ZIP", "type": "text", "source": "overlay", "default": False}, - {"key": "country", "label": "Country", "type": "text", "source": "overlay", - "default": False}, - {"key": "agent", "label": "Sales agent", "type": "text", "source": "overlay", - "default": True, - "description": "The customer's assigned agent (res.partner.agent_ids[0] — the " - "Customers-module convention)."}, - # ⭐ WAVE 28 — the agent's ID beside its NAME, because a link joins on an id and this - # table carried only the display string. ⚠ It is `agent_id`, NEVER `partner_id`: both are - # `res.partner` ids, and joining agents through `JOIN_KEY` would link every customer to - # itself and look plausible doing it. - {"key": AGENT_JOIN_KEY, "label": "Odoo agent id", "type": "int", "source": "overlay", - "default": False}, - _scope_field(), - - # --- the relations ------------------------------------------------------------------- - {"key": "invoices", "label": "Invoices", "type": "link", "source": "overlay", - "default": True, - "link": {"table": INVOICES_KEY, "on": JOIN_KEY, "from": JOIN_KEY}}, - {"key": "orders", "label": "Orders", "type": "link", "source": "overlay", - "default": True, - "link": {"table": ORDERS_KEY, "on": JOIN_KEY, "from": JOIN_KEY}}, - # MEASURED: 2,093 customers carry an `agent_id` and all 2,093 resolve to a row in the - # agents table — zero dangling, which is why this ships as a link rather than a lookup. - {"key": "agent_link", "label": "Agent record", "type": "link", "source": "overlay", - "default": False, - "link": {"table": AGENTS_KEY, "on": AGENT_JOIN_KEY, "from": AGENT_JOIN_KEY}}, - - # --- link rollups over the invoice history -------------------------------------------- - # ⭐ NO CONDITION, and that is measured rather than assumed: a settled document's residual - # is exactly 0, so summing the full history gives the open balance to the cent. - # ⚠ THE LABEL SAYS "ALL CHANNELS" BECAUSE THE COLUMN TOTAL DOES NOT MATCH THE AR PAGE. - # Per customer this is exactly right. Summed down the column it is $2,347,608.49 while - # `Settings → AR` shows $591,828.54 — a 4x gap that is entirely the GIFTWARE DEALS / - # Amazon partner, which wholesale scope excludes and this table deliberately keeps. Two - # numbers with one name, 4x apart, in one product is how a correct figure gets reported - # as a bug; the scope belongs in the label, not only in a column somebody has to filter. - {"key": "ar_outstanding", "label": "AR outstanding $ - all channels", "type": "rollup", - "source": "overlay", "default": True, "agg": "sum", - "description": "Open balance across every posted document, INCLUDING the Amazon " - "channel. Filter `In wholesale scope` to reconcile with the AR page.", - "rollup": {"link": "invoices", "field": "residual", "fn": "sum"}}, - {"key": "invoiced_all_time", "label": "Invoiced $ - all time", "type": "rollup", - "source": "overlay", "default": False, "agg": "sum", - "rollup": {"link": "invoices", "field": "amount_untaxed", "fn": "sum"}}, - # ⛔ THESE TWO DO NEED THE PREDICATE. Over the widened link a bare `countall` counts every - # document ever posted and labels it "open invoices" — the wrong-number-that-looks-right - # this module refuses everywhere else. - {"key": "open_invoices", "label": "Open invoices #", "type": "rollup", - "source": "overlay", "default": True, - "rollup": {"link": "invoices", "fn": "countall", **_OPEN_ONLY}}, - # ⛔ NOT `min`. `_rollup_fold`'s min/max are NUMERIC folds (`_lane_num`), so `min` over a - # date column finds no numbers and returns BLANK — a column that renders empty forever - # while looking configured. Ranking a DATE is what `latest` + `sortBy` is for. - {"key": "oldest_due", "label": "Oldest due date", "type": "rollup", "source": "overlay", - "default": True, - "rollup": {"link": "invoices", "field": "due_date", "fn": "latest", - "sortBy": "due_date", "sortDir": "asc", **_OPEN_ONLY}}, - - # --- link rollups over the order history ---------------------------------------------- - {"key": "order_count", "label": "Orders #", "type": "rollup", "source": "overlay", - "default": True, - "rollup": {"link": "orders", "fn": "countall"}}, - {"key": "last_order", "label": "Last order date", "type": "rollup", "source": "overlay", - "default": True, - "rollup": {"link": "orders", "field": "order_date", "fn": "latest", - "sortBy": "order_date", "sortDir": "desc"}}, - - # --- source-backed (read-through) rollups --------------------------------------------- - # ⛔ THESE DO NOT AND CANNOT COME FROM THE `invoices` LINK. `ut_odoo_invoices` is posted - # BILLING; `revenue_invoiced` is ORDER-LINE revenue narrowed by the order's fully-invoiced - # flag. Different grain, different question — the metric KEY carries the distinction, - # which is the whole reason a rollup may not carry SQL of its own. - {"key": "sales_ytd", "label": "Sales YTD - invoiced", "type": "rollup", - "source": "overlay", "default": True, "agg": "sum", - "rollup": {"source": {"topic": "sales_lines", "measure": "revenue_invoiced", - "groupBy": "order_partner", "on": JOIN_KEY, "window": "ytd"}}}, - {"key": "sales_ltm", "label": "Sales LTM", "type": "rollup", "source": "overlay", - "default": True, "agg": "sum", - "rollup": {"source": {"topic": "sales_lines", "measure": "revenue", - "groupBy": "order_partner", "on": JOIN_KEY, "window": "ltm"}}}, - {"key": "margin_ytd", "label": "Gross margin YTD $", "type": "rollup", "source": "overlay", - "default": False, "agg": "sum", - "rollup": {"source": {"topic": "sales_lines", "measure": "margin", - "groupBy": "order_partner", "on": JOIN_KEY, "window": "ytd"}}}, - {"key": "orders_ytd", "label": "Orders YTD #", "type": "rollup", "source": "overlay", - "default": False, - "rollup": {"source": {"topic": "sales_orders", "measure": "orders", - "groupBy": "partner", "on": JOIN_KEY, "window": "ytd"}}}, - _refreshed_field(), - )] - - -# --------------------------------------------------------------------------------------------- -# READING THE MIRROR -# --------------------------------------------------------------------------------------------- -def excluded_names(): - """The partner names out of wholesale scope, from the ONE place that defines them. - - Read through `core.odoo` rather than re-listed here: a second literal is a second scope, and - the day somebody adds a channel this module would keep answering the old question. - """ - try: - import core.odoo as odoo - names = getattr(odoo, "EXCLUDE_PARTNER_NAMES", None) or set() - return {str(n).strip().lower() for n in names if str(n).strip()} - except Exception: # noqa: BLE001 - return set() - - -def excluded_ids(cur, names=None): - """The out-of-scope partner IDS, resolved against the MIRROR. - - ⭐ IDS, NOT THE DENORMALISED NAME ON THE DOCUMENT, for two reasons that both bite. - `account_move.partner_name` is a copy taken when the document was written, and this module's - own `customers_from` says so out loud — *"a partner's name can differ across documents - (renames land on new invoices only)"*. So a rename would put some of one partner's documents - in scope and the rest out, silently, and the totals would stop reconciling with nothing to - point at. `modules/ar`, the oracle these numbers answer to, has always excluded by ID. - - ⛔ RESOLVED FROM THE MIRROR, NOT `core.odoo.excluded_partner_ids()`. That function issues a - LIVE `search_read`, so importing it here would make spawning four locked databases fail - whenever Odoo is unreachable — including on this developer machine, where the handshake dies - on an expired certificate ([[local-odoo-ssl-quirk]]). Same names, same answer, no network. - - ⚠ MATCHED CASE- AND WHITESPACE-INSENSITIVELY, and a NULL name simply does not match — which - is the correct direction. 44 transacting partners carry no name at all; treating an - unanswerable name as "excluded" would drop $7,734.83 of real open AR out of scope. - """ - names = names if names is not None else excluded_names() - if not names: - return set() - rows = cur.execute("SELECT id, name FROM res_partner WHERE name IS NOT NULL").fetchall() - return {int(pid) for pid, name in rows if str(name).strip().lower() in names} - - -def columns(cur, table): - """The column names a mirror table actually has, lowercased. `set()` if the table is absent. - - ⛔ WHY THIS EXISTS, AND IT COST A LIVE 500. `harness.datastore.ready()` gates on ENTITY - phases, and a Space hydrates its mirror from `store_seed/royal.duckdb` — a SNAPSHOT. Columns - added to `ENTITIES` after that snapshot was taken (`res_partner.agent_id`, - `account_move_line.product_id`, …) are backfilled by `sync_all()` under their OWN `_sync_state` - keys, which `ready()` does not read. So there is a real window, right after a boot, where the - store reports READY and a column this module names does not exist yet — and DuckDB answers a - missing identifier with a Binder error, which reached the operator as a bare `500`. - ⚠ The absent columns are all DISPLAY ones (an agent name, a category, a team). Refusing the - whole spawn over a cosmetic column would be worse than the gap it is reporting, so the readers - degrade the COLUMN to blank and still write every id. - """ - try: - rows = cur.execute(f"SELECT * FROM {table} LIMIT 0") - return {str(d[0]).lower() for d in rows.description} - except Exception: # noqa: BLE001 - return set() - - -def _col(have, name, default="NULL"): - """`name` when the mirror has it, else a literal that keeps the SELECT's arity intact.""" - return name if str(name).split(".")[-1].lower() in have else default - - -def _as_date(value): - """ISO date string, or ''. The grid renders `date` cells itself (W26: `Aug 5, 2026`), so the - STORED value stays ISO — a formatted string in the cell is a value the filters cannot sort.""" - if not value: - return "" - return str(value)[:10] - - -def _in_scope(pid, excluded): - """`'1'` | `''` — the `checkbox` cell convention (`aios_grid`: the overlay stores '1' or '').""" - return "" if int(pid) in excluded else "1" - - -def read_invoices(cur, excluded=None, open_only=False): - """[(row dict)] — posted customer invoices and refunds, keyed on the `account.move` id. - - ONE reader, two projections. `open_only` applies the AR oracle's predicate and drops the - out-of-scope channel, which is what `read_open_ar` wants; the default keeps every row and - TAGS the channel instead. Two queries would be two populations, and they drift the moment - either is edited. - - Takes a CURSOR so a gate can hand it a fixture connection; no global store binding here. - """ - excluded = excluded if excluded is not None else excluded_ids(cur) - where = _AR_WHERE if open_only else _POSTED_DOCS - # ⚠ `invoice_origin` (D-88) is read through `_col` DELIBERATELY. It was added to `ENTITIES` in - # this same wave, so a Space whose mirror is still hydrating from a pre-wave seed snapshot does - # not have the column yet — and DuckDB answers a missing identifier with a Binder error that - # reaches the operator as a bare 500. This is the exact class `columns()` was written for: the - # link degrades to blank for one sync cycle instead of refusing the whole spawn. - have = columns(cur, "account_move") - sql = ("SELECT id, name, partner_id, partner_name, invoice_date, invoice_date_due, " - " amount_untaxed_signed, amount_residual_signed, payment_state, move_type, " - f" {_col(have, 'invoice_origin', chr(39) + chr(39))} " - f"FROM account_move WHERE {where} AND partner_id IS NOT NULL") - out = [] - for r in cur.execute(sql).fetchall(): - (mid, name, pid, pname, inv_date, due, untaxed, residual, pay_state, mtype, origin) = r - scope = _in_scope(pid, excluded) - if open_only and not scope: - continue - out.append({ - "_id": str(mid), - "invoice_no": str(name or ""), - "odoo_id": int(mid), - "customer": str(pname or ""), - JOIN_KEY: int(pid), - "invoice_date": _as_date(inv_date), - "due_date": _as_date(due), - "residual": float(residual or 0.0), - "amount_untaxed": float(untaxed or 0.0), - "payment_state": str(pay_state or ""), - "move_type": str(mtype or ""), - "origin_order": str(origin or "").strip(), - "wholesale_scope": scope, - }) - return out - - -def read_open_ar(cur, excluded=None): - """The OPEN, wholesale-scoped subset — `modules/ar._open_docs`' own population. - - Kept as its own door because `modules/ar` is this module's oracle for the AR numbers, and an - oracle answers exactly one question. It is a projection of `read_invoices`, never a second - query. - """ - return read_invoices(cur, excluded=excluded, open_only=True) - - -def read_orders(cur, excluded=None): - """[(row dict)] — confirmed sale orders, keyed on the `sale.order` id.""" - excluded = excluded if excluded is not None else excluded_ids(cur) - have = columns(cur, "sale_order") - sql = (f"SELECT id, name, date_order, partner_id, partner_name, {_col(have, 'team_name')}, " - f" state, amount_untaxed, {_col(have, 'invoice_status')} " - f"FROM sale_order WHERE {_CONFIRMED} AND partner_id IS NOT NULL") - out = [] - for r in cur.execute(sql).fetchall(): - (oid, name, when, pid, pname, team, state, untaxed, inv_status) = r - out.append({ - "_id": str(oid), - "order_no": str(name or ""), - "odoo_id": int(oid), - "customer": str(pname or ""), - JOIN_KEY: int(pid), - "order_date": _as_date(when), - "amount_untaxed": float(untaxed or 0.0), - "team": str(team or ""), - "state": str(state or ""), - "invoice_status": str(inv_status or ""), - "wholesale_scope": _in_scope(pid, excluded), - }) - return out - - -def read_products(cur): - """[(row dict)] — EVERY `product.product`, keyed on its id. - - ⛔ NO `active` AND NO `default_code` FILTER, and both exclusions were measured before they - were dropped. Filtering to active-and-coded gave 5,829 of 5,948 rows and left FIVE products - that sold this very year with no row at all: two archived SKUs (`9SAT-FY`, `2GSTY`) and three - uncoded charge lines (`UBER CHARGE`, `[Delivery_009] Delivery Charges`, `PICK UP`). A product - grouped by `sales_lines.product` that has no parent row is a rollup value with nowhere to - land — silently. 119 extra rows is the whole cost of the claim being literally true. - """ - have = columns(cur, "product_product") - sql = (f"SELECT id, default_code, name, {_col(have, 'categ_name')}, type, " - f" {_col(have, 'standard_price', '0')}, {_col(have, 'active', 'TRUE')} " - "FROM product_product") - out = [] - for r in cur.execute(sql).fetchall(): - (prid, code, name, categ, ptype, cost, active) = r - out.append({ - "_id": str(prid), - "product": str(name or ""), - PRODUCT_JOIN_KEY: int(prid), - "code": str(code or ""), - "active": "1" if active else "", - "category": str(categ or ""), - "product_type": str(ptype or ""), - "standard_price": float(cost or 0.0), - }) - return out - - -def read_customers(cur, excluded=None): - """[(row dict)] — every CUSTOMER partner, keyed on the `res.partner` id. - - ⭐ THE POPULATION IS A UNION OF THREE LEGS, and every one of them is load-bearing. - - The two DOCUMENT legs are the original pair: Amazon books as direct invoices with no sale - order (the `odoo-api` gotcha), so a sale-order leg alone would silently drop a real customer. - - ⭐⭐ THE THIRD IS `customer_rank > 0 AND active` (wave 29, item 22 / R12 via finding F2 — - the owner's *"never an arbitrary limit… applies to ALL connected database"*). The old - docstring said partners with no document are *"left out on purpose — a row that can never - appear in any topic has nothing to roll up"*; that reasoning is RETIRED. It is the same - join-drop class as the Product grid's 2,717, and it dropped **~1,149 real customer records** - (MEASURED 2026-08-11: `rank>0 active` = 3,617 against a document union of ~2,000). A customer - a salesperson has not sold to yet is exactly the row a prospecting view needs. - - ⛔ IT IS A UNION AND NOT A REPLACEMENT, AND THAT IS MEASURED, NOT TIDINESS. Swapping the - document legs for the rank leg would drop **16 partners that hold posted documents** (7 - archived, 9 active with rank <= 0), and `ut_odoo_invoices` / `ut_odoo_orders` rows carry - `partner_id` LINKS straight back here — so those links would dangle with nothing reporting - it. When other tables point AT a population, a widening must be a SUPERSET. - - ⚠ THE RANK LEG IS SKIPPED WHEN THE MIRROR HAS NO `customer_rank` COLUMN, which is the same - `columns()`/`_col` discipline every other optional column here uses — but note the difference - honestly: an absent `agent_id` blanks a CELL, while an absent `customer_rank` narrows the - POPULATION back to the document union. It degrades to today's behaviour rather than to an - empty or a wrong table, and `verify_odoo_relational` carries a check that goes RED while the - column is missing so the narrowing can never pass for done. - - ⛔ NOT FROM LIVE ODOO, THOUGH `customer_rank` IS TRIVIAL TO ASK IT. `excluded_ids` above - states the rule for this module and it applies with more force to a POPULATION than to a name - list: a live call makes the spawn fail whenever Odoo is unreachable, and a Space hydrates its - mirror from a SNAPSHOT at boot. The population would then be "whichever source answered this - time" — swinging ~45% against `MAX_SHRINK`'s 50% refusal, deleting and re-adding rows on the - weather. One source, always present at spawn time: the mirror. - - ⛔ NOT DERIVED FROM THE INVOICE ROWS. `customers_from` did that when the table WAS the open-AR - partners; sourcing a customer registry from its own receivables is what kept most Odoo ids - out of the store in the first place. - """ - excluded = excluded if excluded is not None else excluded_ids(cur) - have = columns(cur, "res_partner") - # ⚠ THE AGENT JOIN IS DROPPED WHOLE when `agent_id` is absent, not merely NULL-ed: the join - # itself names the column, so `_col` on the SELECT list alone would still fail to bind. - agent = ("ag.name" if "agent_id" in have else "NULL") - agent_id_col = ("p.agent_id" if "agent_id" in have else "NULL") - join = ("LEFT JOIN res_partner ag ON ag.id = p.agent_id " if "agent_id" in have else "") - # ⚠ BOTH columns must be present, not just `customer_rank`: `active` is what keeps an - # archived prospect out, and a rank test without it would re-admit the 47 archived partners - # the mirror carries. Absent ⇒ the leg is dropped WHOLE, exactly like the agent join above. - rank_leg = (" OR (p.customer_rank > 0 AND p.active) " - if {"customer_rank", "active"} <= have else "") - # ⭐⭐ W37-T11 / D-165 — the POSTAL columns join city/state/country here. They ride `_col` like - # every other optional column, so a mirror predating the 2026-08-15 backfill degrades the CELL - # to blank rather than failing the spawn with a Binder error (see `columns()`'s header for the - # live 500 that rule was written after). - sql = (f"SELECT p.id, p.name, {_col(have, 'p.city')}, {_col(have, 'p.state_name')}, " - f" {_col(have, 'p.country_name')}, {agent}, {agent_id_col}, " - f" {_col(have, 'p.street')}, {_col(have, 'p.street2')}, {_col(have, 'p.zip')} " - "FROM res_partner p " - f"{join}" - "WHERE p.id IN (" - f" SELECT partner_id FROM sale_order WHERE {_CONFIRMED} AND partner_id IS NOT NULL " - " UNION " - " SELECT partner_id FROM account_move " - f" WHERE {_POSTED_DOCS} AND partner_id IS NOT NULL)" - f"{rank_leg}") - out = [] - for r in cur.execute(sql).fetchall(): - (pid, name, city, state, country, agent, agent_id, street, street2, zipc) = r - out.append({ - "_id": str(pid), - "customer": str(name or ""), - JOIN_KEY: int(pid), - "street": str(street or ""), - "street2": str(street2 or ""), - "city": str(city or ""), - "state": str(state or ""), - "zip": str(zipc or ""), - "country": str(country or ""), - "agent": str(agent or ""), - AGENT_JOIN_KEY: int(agent_id) if agent_id else "", - "wholesale_scope": _in_scope(pid, excluded), - }) - return out - - -def read_agents(cur): - """[(row dict)] — the UNION of both agent sources, keyed on the `res.partner` id. - - ⛔ `res_partner.agent` is a BOOLEAN and `datastore.BOOL_FIELDS` lists it for a measured reason: - Odoo returns False both for "empty" and for "boolean false", so a bool missing from that list - silently becomes NULL and every row would read "not an agent" indistinguishably from - "unknown". Read it as a truth value, never as a presence test. - """ - have = columns(cur, "res_partner") - if "id" not in have: - return [] - flagged = "p.agent" if "agent" in have else "FALSE" - # ⛔⛔ THE COMMISSION TABLE IS GUARDED AS A **TABLE**, not just as a column, and that - # distinction is the whole point of this block. `columns()` was written for a missing COLUMN - # (a backfill that has not run yet); `account_invoice_line_agent` is an OCA module entity that - # a mirror hydrated from an older seed snapshot may not have AT ALL. A SELECT naming an absent - # table is a DuckDB Binder error, and this reader runs inside `plan()` — so one missing table - # would fail the WHOLE eight-table spawn and reach the operator as a bare 500. That is - # precisely D-107's shape, and it would have arrived on the first deploy of this feature. - # ⚠ DEGRADE, NEVER REFUSE, which is the posture `columns()`'s own docstring sets: without the - # commission table the population falls back to the FLAGGED partners alone and `commissioned` - # reads blank for every row — fewer agents and an honestly empty column, rather than no spawn. - has_comm = bool(columns(cur, "account_invoice_line_agent")) - commissioned = ("(p.id IN (SELECT agent_id FROM account_invoice_line_agent " - " WHERE agent_id IS NOT NULL))" if has_comm else "FALSE") - union_leg = (" SELECT agent_id FROM account_invoice_line_agent WHERE agent_id IS NOT NULL " - " UNION " if has_comm else "") - sql = (f"SELECT p.id, p.name, {flagged}, {commissioned} AS commissioned " - "FROM res_partner p WHERE p.id IN (" - f"{union_leg}SELECT id FROM res_partner WHERE {flagged})") - out = [] - for (aid, name, flag, comm) in cur.execute(sql).fetchall(): - out.append({ - "_id": str(aid), - "agent": str(name or ""), - "odoo_id": int(aid), - AGENT_JOIN_KEY: int(aid), - "flagged": "1" if flag else "", - "commissioned": "1" if comm else "", - }) - return out - - -def read_accounts(cur): - """[(row dict)] — the whole GL chart, keyed on the `account.account` id. - - ⚠ THE EXPENSE PREDICATE IS THE SEMANTIC LAYER'S, copied rather than invented: - `harness/semantic.py`'s `gl_lines` topic scopes expenses as - `account_type in ('expense','expense_depreciation')`. A second definition here is how a - column and a topic start disagreeing about the same word. - ⛔ `account.account` has NO `active` column in this Odoo version (a domain naming it 500s), so - there is nothing to filter and every account is a row. - """ - have = columns(cur, "account_account") - if not have: - return [] - sql = (f"SELECT id, {_col(have, 'code', chr(39) + chr(39))}, " - f" {_col(have, 'name', chr(39) + chr(39))}, " - f" {_col(have, 'account_type', chr(39) + chr(39))} FROM account_account") - out = [] - for (aid, code, name, atype) in cur.execute(sql).fetchall(): - t = str(atype or "") - out.append({ - "_id": str(aid), - ACCOUNT_JOIN_KEY: str(code or ""), - "account_name": str(name or ""), - "odoo_id": int(aid), - "account_type": t, - "is_expense": "1" if t in ("expense", "expense_depreciation") else "", - }) - return out - - -_VENDOR_DOCS = "state = 'posted' AND move_type IN ('in_invoice','in_refund')" - - -def read_bills(cur): - """[(row dict)] — posted vendor bills and refunds, keyed on the `account.move` id.""" - have = columns(cur, "account_move") - if not have: - return [] - sql = ("SELECT id, name, partner_id, partner_name, invoice_date, invoice_date_due, " - f" {_col(have, 'amount_untaxed_signed', '0')}, " - f" {_col(have, 'amount_residual_signed', '0')}, " - f" {_col(have, 'payment_state', chr(39) + chr(39))}, move_type " - f"FROM account_move WHERE {_VENDOR_DOCS} AND partner_id IS NOT NULL") - out = [] - for r in cur.execute(sql).fetchall(): - (mid, name, pid, pname, when, due, untaxed, residual, pay, mtype) = r - out.append({ - "_id": str(mid), - "bill_no": str(name or ""), - "odoo_id": int(mid), - "vendor": str(pname or ""), - VENDOR_JOIN_KEY: int(pid), - "invoice_date": _as_date(when), - "due_date": _as_date(due), - "amount_untaxed": float(untaxed or 0.0), - "residual": float(residual or 0.0), - "payment_state": str(pay or ""), - "move_type": str(mtype or ""), - }) - return out - - -def read_vendors(cur): - """[(row dict)] — every partner carrying a posted vendor bill, keyed on the `res.partner` id. - - ⚠ DERIVED FROM THE BILLS, unlike `read_customers` which is deliberately NOT derived from its - invoices. The asymmetry is intentional and the reason is what that function's own comment - says: a customer registry sourced from receivables is what kept most Odoo ids out of the store. - There is no second document universe for vendors — a partner with no bill has no payable - history to show — so the bill IS the population, and MEASURED it dangles nothing (0 bills - carry a null partner; all 393 vendors resolve in `res_partner`). - """ - have = columns(cur, "res_partner") - if not have or not columns(cur, "account_move"): - return [] - # ⭐⭐ W33-T48 (owner item 13). The grid served FIVE columns off a partner the census measured - # at SEVENTY-SIX populated fields. The nine added here are the ones a buyer actually asks a - # vendor record for — how to reach them, who they are for tax, and where they are. - # ⛔ EVERY ONE GOES THROUGH `_col`, which substitutes a literal when the mirror lacks the - # column. That is not defensive habit: this projection has to keep working against a mirror - # that has not been re-synced since the widening, and the alternative is a reader that raises - # on the exact box the widening was meant to help. The columns arrive when the backfill runs; - # until then these read blank rather than failing. - blank = chr(39) + chr(39) - sql = ("SELECT p.id, p.name, " - f"{_col(have, 'p.country_name', blank)}, {_col(have, 'p.email', blank)}, " - f"{_col(have, 'p.phone', blank)}, {_col(have, 'p.mobile', blank)}, " - f"{_col(have, 'p.website', blank)}, {_col(have, 'p.vat', blank)}, " - f"{_col(have, 'p.ref', blank)}, {_col(have, 'p.street', blank)}, " - f"{_col(have, 'p.street2', blank)}, {_col(have, 'p.city', blank)}, " - f"{_col(have, 'p.zip', blank)} " - "FROM res_partner p WHERE p.id IN " - f" (SELECT partner_id FROM account_move WHERE {_VENDOR_DOCS} " - " AND partner_id IS NOT NULL)") - out = [] - for (pid, name, country, email, phone, mobile, website, vat, ref, - street, street2, city, zipc) in cur.execute(sql).fetchall(): - out.append({ - "_id": str(pid), - "vendor": str(name or ""), - "odoo_id": int(pid), - VENDOR_JOIN_KEY: int(pid), - "country": str(country or ""), - "email": str(email or ""), - "phone": str(phone or ""), - "mobile": str(mobile or ""), - "website": str(website or ""), - "vat": str(vat or ""), - "vendor_ref": str(ref or ""), - "street": str(street or ""), - "street2": str(street2 or ""), - "city": str(city or ""), - "zip": str(zipc or ""), - }) - return out - - -def customers_from(invoice_rows): - """The partners carrying the given invoice rows — the pre-2026-08-09 population builder. - - ⚠ NO LONGER WHAT SPAWNS `ut_odoo_customers` (that is `read_customers`). Kept because it is a - pure function over rows and the gate uses it to prove the FOLD against a fixture without a - mirror; deleting it would cost a test its independence from the SQL. - """ - out = {} - for row in invoice_rows: - pid = row[JOIN_KEY] - entry = out.setdefault(str(pid), {"_id": str(pid), "customer": row["customer"], - JOIN_KEY: pid}) - # A partner's name can differ across documents (renames land on new invoices only); - # the newest non-empty one wins so the locked table shows what Odoo shows today. - if row["customer"]: - entry["customer"] = row["customer"] - return list(out.values()) - - -# --------------------------------------------------------------------------------------------- -# THE SPAWN -# --------------------------------------------------------------------------------------------- -class Refused(Exception): - """A refusal a caller should SHOW, not swallow. Every raise names what would otherwise have - been written wrong.""" - - -#: `plan()` bucket -> (store key, nav label, field contract). ⭐ ONE ROW PER TABLE is the whole -#: point: adding an Odoo entity is a spec row plus a reader, not a fifth copy of the spawn code. -#: ⚠ ORDER MATTERS ONLY FOR THE REFUSAL MESSAGE; `plan` checks every cap before anything commits. -TABLES = ( - # ⛔⛔ `customers` AND `products` ARE GONE FROM THIS TUPLE — W33-T44 / owner item 12 / - # AMENDMENT A2. They presented the same SUBJECTS as the compiled registry modules - # `customer_data` and `product_data` ("why do we have 'Odoo products' already with the current - # Products database? The Unique ID is redundant"), and R2 ruled the LEGACY key survives: it - # keeps its store bucket, so no saved view, grant, cohort or formula moves, and it now carries - # the twins' join keys (`partner_id`; `product_id` pending the ask in `mailbox/E.md`). - # Dropping the row here is what stops them being planned, built or re-created; - # `RETIRED_KEYS` below is what removes the rows a tenant already has. - ("invoices", INVOICES_KEY, "Odoo invoices", invoice_fields), - ("orders", ORDERS_KEY, "Odoo orders", order_fields), - # ⭐ WAVE 28 / R1. Measured populations: 19 / 192 / 6,538 / 393 — every one of them two orders - # of magnitude inside `MAX_ROWS`, which is why the answer to "every unique id is a database" - # is four more spec rows and four readers rather than a new substrate. - ("agents", AGENTS_KEY, "Odoo agents", agent_fields), - ("accounts", ACCOUNTS_KEY, "Odoo GL accounts", account_fields), - ("vendors", VENDORS_KEY, "Odoo vendors", vendor_fields), - ("bills", BILLS_KEY, "Odoo vendor bills", bill_fields), - # ⭐⭐ W30-T35 / R7 — the two READ-THROUGH grains. They are spec rows like any other, and that - # is the point: `apply_plan` creates their DEFINITION (label, fields, lock, nav entry, grants) - # exactly as it does for the eight above, and `plan` hands them ZERO rows. Leaving them out of - # this tuple was the alternative and it is the wrong one — the route 404s on a key `TABLES` - # does not name, so the grids would be bound to the mirror and unreachable, which is this - # wave's own [[reachable-is-not-the-same-as-built]] shape. - ("order_lines", ORDER_LINES_KEY, "Odoo order lines", order_line_fields), - ("gl_lines", GL_LINES_KEY, "Odoo GL lines", gl_line_fields), -) - -#: store key -> the REAL-WORLD POPULATION that table presents (`core.registry`'s `subject` -#: vocabulary, same strings, one namespace). ⭐ W33-T41 / item 12a. -#: -#: ⛔ A SEPARATE MAP RATHER THAN A FIFTH ELEMENT ON EACH `TABLES` ROW, and that is not tidiness: -#: six sites in this file and two in the gate unpack `for bucket, key, _l, _f in TABLES`, so -#: widening the tuple is eight edits that all fail loudly at once and one — in a gate — that -#: would fail QUIETLY, having already been rewritten to match. The subject is a fact ABOUT the -#: key, and this is the shape that says so. -#: -#: ⚠ THE THREE res.partner SUBSETS ARE THREE SUBJECTS, NOT ONE. Customers, agents and vendors all -#: read `res_partner`, and they are different POPULATIONS of it — the claim is over who is in the -#: database, never over which Odoo model was queried. Same for `account_move`, which is the -#: customer-invoice book under one WHERE and the vendor-bill book under another. -TABLE_SUBJECTS = { - # ⛔ `CUSTOMERS_KEY` and `PRODUCTS_KEY` are absent — W33-T44 retired them, and their subjects - # (`odoo:res.partner`, `odoo:product.product`) are claimed by `core.registry`'s `customer_data` - # and `product_data` rows, which is now the ONLY claim on each. That is item 12 satisfied: one - # subject, one database, and `subject_conflict` would REFUSE either of these keys if a future - # spec row tried to bring it back. - INVOICES_KEY: "odoo:account.move.customer", - ORDERS_KEY: "odoo:sale.order", - AGENTS_KEY: "odoo:res.partner.agent", - ACCOUNTS_KEY: "odoo:account.account", - VENDORS_KEY: "odoo:res.partner.vendor", - BILLS_KEY: "odoo:account.move.vendor", - ORDER_LINES_KEY: "odoo:sale.order.line", - GL_LINES_KEY: "odoo:account.move.line", -} - -#: ⛔⛔ THE TWO RETIRED KEYS — W33-T44 / AMENDMENT A2. The rows a tenant ALREADY HAS. -#: -#: Dropping the `TABLES` rows above stops these being planned or re-created; it does NOT remove the -#: definitions and rows already sitting in a tenant's `user_tables` document, which is what the -#: owner actually sees in the nav. This set is what removes them, and it is applied inside -#: `apply_plan`'s single atomic updater — i.e. BY THE CONTAINER, on the boot rebuild and every -#: resync. -#: -#: ⛔ IT MUST BE THE CONTAINER AND NOT A CLI, AND THIS IS MEASURED, NOT CAUTIOUS (D-195): a -#: developer's script CAN write the tenant store, the write returns clean, a fresh read confirms -#: it — and the running Space reverts it within a minute, because the container holds the document -#: and re-uploads its own copy (download-modify-upload, last write wins). A removal shipped as a -#: script is a dry run that reports success. -#: -#: ⛔ AND A SWEEP THAT DELIVERS MUST NOT CREATE. Wave 32's `ut_ensure` was handed a merge-only job -#: and minted 8 empty databases in every tenant, because the door it used creates when absent. This -#: is a `pop`, it runs only over keys already present, and the gate asserts BOTH halves — removed, -#: AND not re-created on the next pass. Gating only the removal would pass on a tree that deletes -#: and re-adds the table every 30 minutes. -RETIRED_KEYS = (CUSTOMERS_KEY, PRODUCTS_KEY) - -#: ⛔ THE GRANDFATHER LIST IS DELETED — W33-T44 did what it was written to force. -#: -#: It existed for exactly one wave, to keep the product runnable between W33-T41 (the uniqueness -#: check) and W33-T44 (the retirement): the check is CORRECT and the collision it forbids was -#: LIVE, so without a named exemption the spawn refused on any fresh document and tenant #0 wrote -#: nothing at all. The exemption was ratcheted BOTH ways — a collision outside it was a new -#: duplicate, a member that stopped colliding was a stale exemption — so retiring the twins turned -#: the gate RED until this constant went with them. It did, and that is the ratchet working. -#: `verify_odoo_relational::_prove_subject_uniqueness` now asserts the collision set is EMPTY. - -#: The bucket -> reader map. ⛔ ITS ABSENCES ARE LOAD-BEARING: a bucket with no reader has no -#: python row builder ANYWHERE, which is what makes "never materialised" structural rather than a -#: policy `plan()` could forget. The two line grains are absent for that reason and no other. -_READERS = { - "customers": lambda cur, excluded: read_customers(cur, excluded=excluded), - "products": lambda cur, excluded: read_products(cur), - "invoices": lambda cur, excluded: read_invoices(cur, excluded=excluded), - "orders": lambda cur, excluded: read_orders(cur, excluded=excluded), - "agents": lambda cur, excluded: read_agents(cur), - "accounts": lambda cur, excluded: read_accounts(cur), - "vendors": lambda cur, excluded: read_vendors(cur), - "bills": lambda cur, excluded: read_bills(cur), -} - -#: The table keys this module can never materialise — DERIVED from the absence of a reader, never -#: typed out, so it cannot drift from the fact it describes. -#: -#: ⛔⛔ IT IS STAMPED ONTO THE DEFINITION AT SPAWN, AND THAT IS NOT BELT-AND-BRACES — IT IS THE -#: ONLY WAY THESE TWO TABLES EVER GET THE DURABLE FLAG. `core.user_tables.materialises` reads a -#: process-global registry first and falls back to a stored `readThrough` stamp, "which is what a -#: cold process reads" — but the only writer of that stamp is `strip_materialised`, and it stamps -#: exclusively tables it found rows on (`if isinstance(t, dict) and t.get('rows')`, after an early -#: return when nothing is fat). A table that was BORN read-through has no rows to strip, so it is -#: never stamped, so a process that cannot reach the mirror reads `rows: {}` and calls that the -#: answer — an EMPTY GRID with nothing going red, which is the exact failure that docstring names. -#: The conversion writes the stamp; a table that needs no conversion still needs the statement. -READ_THROUGH_KEYS = frozenset(key for bucket, key, _l, _f in TABLES if bucket not in _READERS) - - -class _LentDoc: - """A store handle that serves the ONE `user_tables` document `plan()` has ALREADY read. - - ⛔⛔ THIS IS NOT A MICRO-OPTIMISATION AND IT IS NOT OPTIONAL. `core.user_tables.row_limit` - resolves through `materialises` → `get` → `all_tables(st)`, and every one of those is a WHOLE - 20 MB document read, deep-copied under `Store._lock`. `plan()` asks the evaluator once per - table per loop, so passing the live handle would have added ~16 full document copies to a - function that already reads it exactly once — and with `st=None` (the gate's fixture posture, - and any dry run) those reads resolve to the MODULE-GLOBAL store, i.e. a Hugging Face dataset - fetch per table, on a path that has no business touching the network at all. - `materialises`' own docstring asks callers to lend the definition they are holding; `row_limit` - takes `st` rather than `defn`, so the lending happens one level up, here. - - ⚠ It answers ONLY the user-tables document and `None` for anything else, deliberately: a shim - that quietly proxied other keys would be a second store with a partial view, which is worse - than one that says what it knows. - """ - - def __init__(self, doc, key): - self._doc, self._key = doc if isinstance(doc, dict) else {}, key - - def get(self, name): - return self._doc if name == self._key else None - - -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# ⭐⭐ W32-T15/T16/T17 / CONTRACT C2 / RULINGS R9, R10, R11 — THE CONNECTOR'S OWN CONFIGURATION -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# -# The owner opened the Odoo connector and found nothing to configure: no key, no server database, -# no choice of which grids to materialise, no sync cadence, no way off. R9/R10/R11 answer all -# four, and the state lives HERE rather than in the route because `refresh()` is what has to obey -# it — a config the route knows and the sync path does not is a switch that flips nothing. -# -#: `{"grids": {table_key: bool}, "syncEvery": "", "frozen": bool, "frozenAt": ""}` -CONFIG_KEY = "odoo_connector_config" - -#: ⭐ R11's presets, and the FLOOR IS THE POINT. Owner: *"30m / 1h / 4h / daily / manual. The -#: floor is 30 minutes, server enforced; no free-text interval."* An interval box would let a -#: tenant ask for 60 s against an ERP over XML-RPC and a Hugging Face free-tier container. -#: ⚠ `manual` is not "very slow" — it is NO scheduled sync at all, which is why it maps to None -#: rather than to a large number. A caller that treats None as a duration gets a TypeError rather -#: than a silent once-a-century schedule. -SYNC_PRESETS = {"30m": 1800, "1h": 3600, "4h": 14400, "daily": 86400, "manual": None} -SYNC_FLOOR_SECONDS = 1800 -DEFAULT_SYNC = "30m" - - -def read_config(rt): - """This tenant's stored connector config, defaulted. Never raises — a store hiccup must not - make a connector look disconnected.""" - try: - cur = rt.get(CONFIG_KEY) if rt is not None else None - except Exception: # noqa: BLE001 - cur = None - cur = cur if isinstance(cur, dict) else {} - grids = cur.get("grids") if isinstance(cur.get("grids"), dict) else {} - every = cur.get("syncEvery") - return {"grids": {str(k): bool(v) for k, v in grids.items()}, - "syncEvery": every if every in SYNC_PRESETS else DEFAULT_SYNC, - "frozen": bool(cur.get("frozen")), - "frozenAt": str(cur.get("frozenAt") or "")} - - -def grid_choices(rt): - """`[{key, label, enabled}]` for every grid this connector can materialise — DERIVED from - `TABLES`, never a second hand-typed list (contract C2's parity leg asserts exactly that). - - ⚠ ABSENT MEANS ENABLED. A tenant that has never opened the panel has every grid, which is - what they have today; only an explicit untick turns one off. The alternative — an empty - config meaning "nothing enabled" — would silently unspawn ten live databases on deploy. - """ - chosen = read_config(rt)["grids"] - return [{"key": key, "label": label, "enabled": bool(chosen.get(key, True))} - for _bucket, key, label, _fields in TABLES] - - -def enabled_buckets(rt): - """The BUCKET names `plan()` speaks, for the grids this tenant has left ticked.""" - chosen = read_config(rt)["grids"] - return {bucket for bucket, key, _l, _f in TABLES if chosen.get(key, True)} - - -def sync_seconds(rt): - """How often this tenant's Odoo mirror should resync, or None for `manual` (R11). - - ⛔ THE FLOOR IS ENFORCED HERE AS WELL AS AT THE WRITE DOOR, deliberately. A stored value that - predates the preset list, or one written by any path that is not the route, must still not be - able to ask this loop for a 60-second cycle — a limit with only one enforcer is a limit that - holds until somebody finds the second way in [[limit-with-no-enforcer]]. - """ - secs = SYNC_PRESETS.get(read_config(rt)["syncEvery"], SYNC_PRESETS[DEFAULT_SYNC]) - if secs is None: - return None - return max(int(secs), SYNC_FLOOR_SECONDS) - - -def frozen(rt): - """Has this tenant DISCONNECTED Odoo (R10)? Frozen grids keep every row and every field and - stop being refreshed — distinct from PAUSED, which is temporary and keeps the credential.""" - return bool(read_config(rt)["frozen"]) - - -def plan(cur, rt=None): - """The rows that WOULD be written, plus the refusals that apply — no store WRITE at all. - - Separated from `apply_plan` so a route, a gate and a dry run all measure the same thing, and - so **every cap is checked before anything is committed**. - - Returns one key per bucket plus two that are not buckets: `problems` (refusals — a non-empty - list makes `apply_plan` raise before it writes anything) and, since W30-T35, **`limits`** — - `{table_key: limit_report}` for every table this plan did not fully materialise, which is R6's - second sentence carried as data rather than left for a reader to infer from an empty list. - - ⛔ `rt` IS WHAT MAKES THE TABLE-COUNT CHECK HONEST, and leaving it out was a real half-spawn - bug. This spawn writes FOUR tables in four updater passes; a tenant near `MAX_TABLES` would - create some and refuse the rest — leaving a locked invoices database with no rollup host, - while the route answered as though nothing had happened. A partial spawn is worse than a - refused one, so the count is checked against the tables that ALREADY exist, before the first - write. `rt` also carries the stored row counts the shrink guard compares against. - """ - ut = _ut() - # ⭐ R6, AND WITHOUT THIS LINE THE RULE IS ONLY ACCIDENTALLY TRUE. `is_connected` answers from - # three places in falling authority: the registry, a stored `connected: True`, then the - # `ut_odoo_` naming convention — and that last leg needs the table to ALREADY EXIST. So on a - # FIRST spawn, in a process that has not yet built `routes_odoo_tables.GRID_SOURCES`, every one - # of these tables reads as unconnected and earns `MAX_ROWS`: R6's cap removal would silently - # not apply on exactly the run that creates the databases. This module DECLARES these keys, so - # it is the honest place to say what they are. Idempotent (a set add), and it fills the - # evaluator's input rather than becoming a second evaluator. - ut.register_connected(*[key for _b, key, _l, _f in TABLES]) - excluded = excluded_ids(cur) - # ⭐⭐ W30-T35 / R6 / R7 — WHICH BUCKETS ARE BUILT AT ALL IS NOW ASKED, NOT ASSUMED, and it is - # `core.user_tables.row_limit` that answers: 0 = "stores no rows HERE" (read-through), None = - # "connected and uncapped", MAX_ROWS = "the editable substrate". Its own docstring names this - # function as the caller that reads it, which is the seam working as designed — one evaluator, - # so the spawn, the write doors and the wire cannot disagree about whether a table is capped - # ([[one-evaluator-per-question]]). - # - # ⛔ TWO DIFFERENT REASONS NOT TO BUILD, AND THEY ARE KEPT SEPARATE ON PURPOSE: - # * no reader at all — structural, permanent, and the case that must not depend on a store - # read succeeding (a cold process with no mirror still must not try to build 963,783 rows); - # * a reader exists but the table has already been converted to read-through — the - # `ut_odoo_accounts` case. Building 192 rows and letting `strip_materialised` delete them - # again on the next pass "works", and it is exactly the wasted, dangerous work `row_limit` - # was built to prevent. It also stops a refresh from silently RE-MATERIALISING a table - # D-87's conversion had already emptied. - # - # ⚠ THE DOCUMENT IS READ **ONCE**, HERE, AND LENT TO THE EVALUATOR — see `_LentDoc`. It used - # to be read after the build loop; it moved up because `row_limit` needs it and reading it per - # table per loop is ~16 more whole-document deep copies (or, with `st=None`, a Hugging Face - # fetch per table on a path that must never touch the network). - existing = {} - if rt is not None: - try: - existing = dict(rt.get(ut.STORE_KEY) or {}) - except Exception: # noqa: BLE001 - existing = {} - # ⛔ THE LENT DOCUMENT CARRIES THE `readThrough` STAMP THIS MODULE IS RESPONSIBLE FOR, and - # without it R6's report is silently absent on the run that matters most — the FIRST spawn. - # MEASURED: with an empty store, `row_limit` finds no registry entry and no stored stamp, so - # it answers `None` ("connected and uncapped") for a grain that stores nothing at all, and - # `limit_report` answers None with it — so `plan()["limits"]` came back EMPTY and the grids - # were skipped with no stated reason. The structural `reader is None` guard still did its job; - # what went missing was the half of R6 that has to SAY WHY. - # - # ⚠ THE OBVIOUS FIX IS THE ONE I DID NOT TAKE: `ut.register_read_through(*READ_THROUGH_KEYS)` - # would work in one line, and `core.user_tables` explicitly reserves that registrar — - # *"IT IS ALSO THE ONLY PLACE THAT MAY CALL `register_read_through`"* — for - # `routes_odoo_tables.sync_read_through`, because ELIGIBILITY needs the mirror and the fold - # matrix. That reasoning does not apply to a grain with no reader (there is nothing it could - # be eligible FOR), but the law is written without an exception, so this lends the evaluator - # the definition instead of taking one. `_ensure_table_inplace` writes exactly this stamp, so - # what is lent is the document as it stands the moment this plan applies. - lent = _LentDoc({**existing, - **{k: {**(existing.get(k) or {}), "readThrough": True} - for k in READ_THROUGH_KEYS}}, - ut.STORE_KEY) - - built, limits = {}, {} - caps = {} - for bucket, key, _label, _fields in TABLES: - reader = _READERS.get(bucket) - caps[key] = cap = ut.row_limit(key, st=lent) - if reader is None or cap == 0: - built[bucket] = [] - report = ut.limit_report(key, st=lent) - if report: - limits[key] = report - continue - built[bucket] = reader(cur, excluded) - problems = [] - - if rt is not None: - needed = [k for _b, k, _l, _f in TABLES if k not in existing] - if needed and len(existing) + len(needed) > ut.MAX_TABLES: - problems.append( - f"tenant holds {len(existing)} of MAX_TABLES={ut.MAX_TABLES} user tables and " - f"needs {len(needed)} more ({', '.join(needed)}); refusing rather than creating " - f"part of a linked set") - - for bucket, key, _label, _fields in TABLES: - rows = built[bucket] - cap = caps[key] # asked ONCE per table, above — never re-read per loop - # ⭐⭐ R6: THE `MAX_ROWS` REFUSAL IS GONE FOR A CONNECTED SOURCE, AND THE SENTENCE IT USED - # TO PRINT IS NOW `limit_report`'s STRUCTURED ANSWER. Owner, verbatim: *"there is no cap in - # how many data from the API source (as long as its from a connected source like Odoo) that - # can be pulled into the app… Now if there is lag or it can't be done, you need to - # explicitly tell me why and recommend a fix."* Both halves are here: a connected table - # answers `None` and is never refused for its size, and anything that IS still bounded is - # reported with its cause and its recommendation instead of a hand-typed line. - # - # ⛔ THE `cap and` GUARD IS THE WHOLE CHANGE AND ITS TWO FALSY CASES MEAN OPPOSITE THINGS: - # `None` = connected, uncapped, build every row Odoo has; `0` = stores no rows here, and - # the loop above already handed it an empty list. Neither may reach the refusal. The - # editable substrate still gets `MAX_ROWS` and is still REFUSED, never truncated — a capped - # table understates every total it feeds while looking exactly like a complete one. - if cap and len(rows) > cap: - report = ut.limit_report(key, st=lent) or {} - limits[key] = report - problems.append( - f"{key}: {len(rows):,} rows exceeds the {cap:,}-row ceiling; refusing " - f"({report.get('cause', 'a truncated table understates every rollup it feeds')}). " - f"{report.get('recommendation', '')}".strip()) - # ⚠ THE SHRINK GUARD SKIPS A READ-THROUGH GRAIN, and without this it would refuse every - # spawn after the first conversion: zero rows against a stored population is the INTENDED - # end state there, not the partial mirror read this guard exists to catch. - if cap == 0: - continue - stored = len(((existing.get(key) or {}).get("rows")) or {}) - if stored and len(rows) < stored * MAX_SHRINK: - problems.append( - f"{key}: the mirror answered {len(rows)} rows against {stored} stored — a drop of " - f"more than {int((1 - MAX_SHRINK) * 100)}% is a bad read, not Odoo history " - f"shrinking; refusing rather than deleting rows that still exist") - built["problems"] = problems - # R6's second sentence as DATA rather than prose: every table whose rows this plan did not - # (or may not) materialise, with the cause and the recommendation `core.user_tables` derives. - # ⚠ NOT a bucket — `apply_plan` iterates `TABLES` and asks `if bucket in built`, so a key that - # is not a bucket name is inert there, exactly as `problems` has always been. - built["limits"] = limits - return built - - -def apply_plan(rt, built, username="automation", today=None, report=None): - """Create-or-merge every table in `built` and its rows. Idempotent by construction. - - Row ids ARE the Odoo ids, so a re-run updates in place and never appends a second copy of the - same record — which is also what makes "every Odoo unique id is in the database" a checkable - statement rather than a hopeful one. - - ⚠ ONLY THE BUCKETS PRESENT ARE WRITTEN, so a caller (or a gate) may hand in a subset. - """ - if built.get("problems"): - raise Refused("; ".join(built["problems"])) - stamp = today or _iso_today() - written = {} - # ⚠ An OPTIONAL out-parameter, not a return-shape change: `written` has one value shape and - # keeps it. A caller that wants to SAY what was retired passes a dict; `refresh` does. - retired = [] - plans = [(key, label, fields(), built[bucket]) - for bucket, key, label, fields in TABLES if bucket in built] - - # ⭐⭐ ONE SYNC WRITE FOR ALL FOUR TABLES, not one per table — measured, not tidied. - # - # ⛔ A `flush="sync"` update of `user_tables` is a FULL DOWNLOAD of the document plus a full - # UPLOAD of it (`Store.update` -> `_read_strict` -> `put`). Four of them against the 20.6 MB - # document these tables produce is ~165 MB of Hugging Face traffic and four dataset commits - # EVERY resync — and `main.py` runs this at boot and after every `sync_all()` (~30 min). - # Composed into one pass it is ~41 MB and one commit: the same rows, a quarter of the bill. - # - # ⭐ AND IT IS ATOMIC, WHICH IS THE BIGGER WIN. `_ensure_table_inplace` raises `Refused` at - # `MAX_TABLES`; with four separate writes that refusal landed AFTER earlier tables had - # already been committed, leaving exactly the half-spawn `plan()` opens by refusing to - # create. Inside one updater, a raise aborts before anything is persisted. - def _apply_all(cur): - cur = cur if isinstance(cur, dict) else {} - # ⛔⛔ W33-T44 — THE RETIREMENT, INSIDE THE SAME ATOMIC WRITE. This updater is the ONE - # sync write the whole spawn makes, and it runs in the CONTAINER (boot rebuild + every - # resync), which is the only place a change to the tenant document survives (D-195: the - # same edit from a CLI reports success and is reverted within a minute). - # ⚠ It POPS and never creates: `RETIRED_KEYS` are absent from `TABLES`, so `plans` cannot - # contain them, and there is no door here that could re-add one. `removed` is reported so - # a caller can SAY what happened rather than infer it from a table going missing. - # ⛔ THE REMOVED KEYS DO **NOT** GO INTO `written`, and the first cut of this put them - # there. `written` maps table key -> a COUNTS dict, and every consumer iterates its - # `.values()` expecting `c["added"]`; a list under `"_retired"` made the very next leg die - # with `TypeError: list indices must be integers`. One dict, two value shapes, is a - # sentinel in a result set [[sentinel-in-a-sort-key]] — so the retirement reports through - # its OWN channel and the return type stays exactly what it was. - for key in RETIRED_KEYS: - if cur.pop(key, None) is not None: - retired.append(key) - for key, label, fields, rows in plans: - written[key] = _ensure_table_inplace(cur, key, label, fields, rows, username, stamp) - return cur - - rt.update(_ut().STORE_KEY, _apply_all, flush="sync") - if report is not None and retired: - report["retired"] = sorted(retired) - return written - - -def _ensure_table(rt, key, label, fields, rows, username, stamp): - """One table, written on its own. Kept because the gate drives a single table directly, and - because a caller with one table to reconcile should not have to compose an updater.""" - written = {} - - def _one(cur): - cur = cur if isinstance(cur, dict) else {} - written["counts"] = _ensure_table_inplace(cur, key, label, fields, rows, username, stamp) - return cur - - rt.update(_ut().STORE_KEY, _one, flush="sync") - return written["counts"] - - -def _ensure_table_inplace(cur, key, label, fields, rows, username, stamp): - """One table INSIDE a caller's updater: definition merged, rows reconciled, dict mutated. - - ⚠ ROWS THAT LEFT THE POPULATION ARE REMOVED, and that stayed correct through the widening — - but only because the populations widened to "everything Odoo has". While `ut_odoo_customers` - was built FROM open invoices, removal meant a customer who paid their bill vanished from the - registry; now a partner leaves only when their last document does. The shrink guard in - `plan()` is the backstop for the case this policy cannot distinguish: a partial mirror read. - """ - ut = _ut() - wanted = {r["_id"]: {k: v for k, v in r.items() if k != "_id"} for r in rows} - for row in wanted.values(): - row["refreshed"] = stamp - counts = {"added": 0, "updated": 0, "removed": 0, "rows": len(wanted)} - subject = TABLE_SUBJECTS.get(key) - table = cur.get(key) - if table is None: - if len(cur) >= ut.MAX_TABLES: - # ⚠ `ut_ensure` returns silently at this cap; a silent no-op here would report a - # successful refresh over a table that does not exist. - raise Refused(f"{key}: tenant is at MAX_TABLES={ut.MAX_TABLES}; nothing created") - # ⭐⭐ W33-T41 / item 12a — THE UNIQUENESS CHECK, ON THE LINE THAT MINTED THE DUPLICATE. - # - # ⛔ IT GUARDS **CREATION ONLY**, AND THAT IS THE WHOLE DESIGN, NOT A WEAKENING. This - # function is the boot rebuild and the 1800 s resync; it adopts an existing table by key - # on every pass. A claim test outside this `if` refuses the table it created last boot, - # `_apply_all` raises inside the updater, and tenant #0 spawns NOTHING — the check would - # take the product down to prevent a duplicate that already exists. Refusing the SECOND - # birth is what "make sure this never happens" asks for; the FIRST one is T44's job to - # remove, and it is removed by dropping its `TABLES` row, not by a guard here. - # - # ⚠ Claims are gathered from THIS TENANT'S document (`cur`) plus the compiled registry. - # Never a module-level cache: one Space process serves every tenant, and two tenants both - # holding an "Odoo customers" is correct (that is D-169's shape, and it is not repeated - # here). A stored definition with no `subject` claims nothing. - held = {t["subject"]: k for k, t in cur.items() - if isinstance(t, dict) and t.get("subject")} - # ⭐ NO EXEMPTION ANY MORE. The grandfather list is deleted with the two tables it covered - # (W33-T44), so this is now the plain rule the owner asked for: one subject, one database. - other = _registry().subject_conflict(subject, key, claimed=held) - if other: - raise Refused( - f"{key}: refusing to create a second database for {subject!r} — {other!r} " - f"already presents it. One subject, one database (item 12a); if this table is " - f"meant to replace {other!r}, retire {other!r} first rather than shipping both") - table = cur[key] = { - "key": key, "label": label, "source": ut.AUTOMATION_SOURCE, - "createdBy": username, "created": stamp, "fields": [], "rows": {}, - # recordMode = a LOCKED database (item-3 nomenclature): no human may add or - # delete records, while fields stay addable. Odoo owns this population. - "recordMode": ut.AUTOMATION_RECORD_MODE, - } - table.setdefault("recordMode", ut.AUTOMATION_RECORD_MODE) - # W30-T35 — the durable "my rows are not in this document" statement, on the tables no - # conversion will ever stamp (see `READ_THROUGH_KEYS`). Written on every pass, not - # `setdefault`: it is derived from the code's own structure, so the code is what it must agree - # with, and a definition that somehow lost the flag should regain it rather than keep serving - # an empty grid. - if key in READ_THROUGH_KEYS: - table["readThrough"] = True - # W33-T41 — the claim, made DURABLE on the definition. Written on every pass for the same - # reason `readThrough` is: it is derived from this module's own structure, so the code is what - # it has to agree with, and a definition that lost the stamp should regain it rather than go - # on being invisible to the next table's claim test. ⚠ It is also the ONLY way the check sees - # a `ut_*` table at all — those are per-tenant DATA, never compiled registry rows, so a cold - # process reading a fresh document has nothing else to read the claim off. - if subject: - table["subject"] = subject - have = {str(f.get("key")): f for f in (table.get("fields") or [])} - for field in fields: - # ⛔⛔ THE ONE DOOR THAT BYPASSES THE FIELD VALIDATOR, HARDENED WHERE IT BYPASSES IT. - # - # `user_tables._clean_field` stamps `source: 'overlay'` unconditionally on create AND - # patch, so every field that goes through the normal door has one. This function does NOT - # go through that door — it writes definitions straight into the document — and - # `aios_grid.rows_from_pool` reads `field["source"]` as a HARD KEY, so a contract that - # ever omitted it would 500 the entire rows route rather than degrade one column. Measured - # today across 9 automation-owned tables and 173 fields: zero are missing it, so this is - # LATENT, not live (A's PENDING row, raised as D-9 and corrected by D-23 — the crash that - # prompted it came from a hand-written test fixture, not from production). - # - # ⚠ Fixed HERE rather than by softening `rows_from_pool` to `.get`, deliberately: a - # missing `source` means the definition does not say which stratum owns the column, and - # rendering it as blank would bury that. The default matches what the validator would - # have stamped, so the bypass stops being a hole without inventing a second rule. - if isinstance(field, dict) and not field.get("source"): - field = {**field, "source": "overlay"} - fkey = str(field.get("key")) - if fkey not in have: - table.setdefault("fields", []).append(dict(field)) - continue - # ⭐ A PRESET FIELD'S CONTRACT IS FORWARD-MIGRATED, not merely created once. The - # widening moved `payment_state`'s option list and every rollup's conditions; a - # create-only merge would have left the LIVE table declaring the old contract - # forever, so the column would render but its filter could not match what is stored. - # ⚠ Only machine-owned keys are touched — `automation.preset` is the wall — so a - # column a user added to a locked database is never rewritten. - stored = have[fkey] - # ⭐⭐ 2026-08-09 — a column a human has taken over keeps its own definition. Same stamp, - # same reader (`user_tables.user_edited`) and the same reason as the IG reconciler: the - # loop below overwrites `rollup` from the shipped contract, so an edited preset rollup on - # an Odoo database would silently revert at the next boot rebuild. - if _ut().user_edited(stored): - continue - if (stored.get("automation") or {}).get("preset"): - for prop in ("label", "type", "options", "link", "rollup", "description", - "agg", "pinned", "default"): - if prop in field: - stored[prop] = field[prop] - else: - stored.pop(prop, None) - stored_rows = table.setdefault("rows", {}) - for rid, values in wanted.items(): - current = stored_rows.get(rid) - if current is None: - stored_rows[rid] = dict(values) - counts["added"] += 1 - elif any(str(current.get(k, "")) != str(v) for k, v in values.items() - if k != "refreshed"): - current.update(values) - counts["updated"] += 1 - else: - current["refreshed"] = values["refreshed"] - for rid in [r for r in stored_rows if r not in wanted]: - stored_rows.pop(rid, None) - counts["removed"] += 1 - return counts - - -def is_royal(tenant): - return str(tenant or "").strip().lower() in RI_SLUGS - - -def refresh(rt, tenant, username="automation", cur=None, today=None): - """THE entry point — the store-resync path and the route both call this. - - ⚠ It must be CALLED on resync by something outside this file. If it is not wired, every row - still carries a `refreshed` stamp, so a stale worklist is at least LEGIBLE rather than - silently authoritative. - """ - if not is_royal(tenant): - raise Refused(f"tenant {tenant!r} has no Odoo mirror behind these tables (R1: Royal " - f"Imports only); refusing to spawn empty locked databases") - # ⭐⭐ W31-T45 / D-169 — THE SLUG GATE ABOVE AND THE FILE GATE HERE ANSWER DIFFERENT QUESTIONS, - # and this is the one place in the codebase where that is easy to miss. `is_royal` asks "is - # this tenant ENTITLED to Odoo databases"; it says nothing about WHICH DuckDB file this process - # has open. A worker pinned to another tenant's store (AIOS_DUCKDB_PATH, or a `use_path` in a - # provisioning script) passes `is_royal("royal-imports")` and then WRITES tenant #0's locked - # databases from another customer's rows — a spawn, not a read, so the wrong numbers become - # durable. Entitlement is not residency. - # ⚠ It runs when `cur` is LENT too, not only when we open one: the resync loop and the boot - # rebuild both hand a cursor in, and a lent cursor is exactly the case where nobody re-checks. - if rt is not None: - rt.assert_datastore_matches() - # ⛔⛔ W32-T16 / R10 — A DISCONNECTED CONNECTOR DOES NOT REFRESH, AND THAT IS THE WHOLE FREEZE. - # R10: *"Disconnect removes the credential and FREEZES the grids as static data."* Removing - # the credential alone is not a freeze — this function is also reached by the boot rebuild and - # the resync loop, and for tenant #0 the ENVIRONMENT still holds Odoo credentials, so a - # disconnected workspace would silently re-materialise from `.env` on the next tick and the - # "disconnect" would last until the container restarted. The refusal is a REPORT, not a raise: - # the resync loop calling this every cycle must not be handed an exception as a status. - if rt is not None and frozen(rt): - return {"tables": {}, "frozen": True, - "note": "this workspace has disconnected Odoo; its databases are frozen as " - "static data and are not being refreshed"} - if cur is None: - from harness import datastore - cur = datastore.ro_con() - built = plan(cur, rt=rt) - # ⭐ W32-T15 / R9 — THE GRID PICKER, ENFORCED WHERE IT COUNTS. `apply_plan` writes only the - # buckets present in `built`, so dropping an unticked one here is the whole of "unticking a - # grid stops it materialising on the next sync". Done AFTER `plan` rather than inside it so - # every cap, refusal and limit report is still computed over the full set — a config must not - # be able to hide a problem by hiding the table that has it. - # ⚠ It does NOT delete a grid that was already spawned. Unticking stops the next refresh from - # rewriting it; dropping the rows a tenant already has is `disconnect`'s job, and it does not - # do that either (R10 keeps them). Silent data deletion behind a checkbox is not on offer. - if rt is not None: - keep = enabled_buckets(rt) - skipped = sorted(key for bucket, key, _l, _f in TABLES if bucket not in keep) - for bucket, _key, _l, _f in TABLES: - if bucket not in keep: - built.pop(bucket, None) - else: - skipped = [] - # ⭐ W33-T44: a table this pass REMOVED is reported, not inferred from a grid going missing. - # Same rule as `skipped` below — R6's second sentence: what was deliberately not built (or no - # longer built) SAYS SO, with the keys. - plan_report = {} - written = apply_plan(rt, built, username=username, today=today, report=plan_report) - return {"tables": written, - **({"retired": plan_report["retired"]} if plan_report.get("retired") else {}), - # R6's second sentence: a set that was deliberately not built SAYS SO, with the keys. - **({"skipped": skipped} if skipped else {}), - **{bucket: len(built[bucket]) for bucket, _k, _l, _f in TABLES if bucket in built}} +"""odoo_relational.py — Odoo entities as LOCKED relational databases. + +Owner ruling R1 / contract C8: spawn preset Odoo databases for Royal Imports, give them preset +Link + Rollup fields, and prove each rollup against the `measure_` column it will eventually +replace. + +⭐⭐ 2026-08-09 — THE POPULATIONS WIDENED FROM "OPEN AR" TO **EVERY ODOO ID**, which is the +owner's item: *"make sure we have all the Unique ID in Odoo in Database for the Royal Imports +tenant."* Before this change there were two tables holding 438 partners and 1,228 invoices — the +partners who owed money — so most Odoo ids were simply absent, and the missing rows were the +reason a Rollup could not answer a sales question. Four tables now, keyed on the Odoo id itself: + + ut_odoo_customers 2,465 rows 0.43 MB every partner with a confirmed order or a + posted customer document + ut_odoo_products 5,829 rows 1.20 MB every active product carrying a SKU code + ut_odoo_invoices 31,418 rows 8.68 MB EVERY posted customer invoice + refund + ut_odoo_orders 32,700 rows 7.50 MB every confirmed sale order + +⛔ WHAT MADE THAT LEGAL, AND IT WAS NOT A BIGGER NUMBER. `MAX_ROWS` was 5,000 and this module's +own `plan()` refused above it — but the cap was never a property of the store (see the measured +banner on `core.user_tables.MAX_ROWS`; `ig_master` has run a 500,000-row bucket the whole time). +The cap is now 60,000, DERIVED from what a row actually weighs (⚠ this line said 100,000 until +wave 28 — that was the FIRST candidate and its own derivation REJECTED it for clearing the memory +budget by 0.6%; the prose was written before the number lost, and two sibling files said it too). +The four tables together are +17.81 MB in one `user_tables` document — real, bounded, and booked: the per-table row-key split +is D-87's next increment. ⛔ ORDER LINES REMAIN OUT (256,810 rows / 63.9 MB / 2.57 s per copy); +they are answered by the read-through rollup, which never copies a row. + +⭐ THE EXCLUDED CHANNEL IS NOW A COLUMN, NOT A DELETION. `core.odoo.EXCLUDE_PARTNER_NAMES` puts +GIFTWARE DEALS (partner 6369 — the Amazon channel) outside WHOLESALE scope, and the old tables +dropped its rows entirely. Dropping them contradicts "every Odoo id", so the rows are kept and +carry **`wholesale_scope`** instead. ⚠⚠ READ THIS BEFORE COMPARING ANY TOTAL: that one partner +holds **$1,755,779.95 of the $2,347,608.49** raw open balance — 75% of it — across 25 invoices. +Wholesale open AR is $591,828.54. So a column total here will not equal the AR page unless you +filter `wholesale_scope`, and that is the scope difference, not a defect. `read_open_ar` keeps +excluding, because `modules/ar` is its oracle and an oracle answers ONE question. + +⭐ AR SURVIVED THE WIDENING UNCHANGED, AND THAT IS MEASURED, NOT ASSUMED. `sum(residual)` over +ALL posted customer documents equals `sum(residual)` over `modules/ar._open_docs`' own predicate +**to the cent** ($2,347,608.49): zero posted rows carry a non-zero residual outside +`payment_state IN ('not_paid','partial')`, and zero rows inside it carry a residual of 0. So the +`ar_outstanding` rollup needs no condition. ⛔ THE COUNT AND THE DATE DO — `countall` over the +wider link would count 31,418 documents and call them open invoices, so those two rollups carry +the oracle's predicate as an explicit `payment_state` condition pair. +""" +import datetime as _dt + +#: Royal Imports only (R1). A tenant slug that is not this one gets a refusal, never a spawn: +#: nurilab has no Odoo mirror behind these tables and would get empty locked databases. +RI_SLUGS = ("", "royal-imports") + +INVOICES_KEY = "ut_odoo_invoices" +CUSTOMERS_KEY = "ut_odoo_customers" +ORDERS_KEY = "ut_odoo_orders" +PRODUCTS_KEY = "ut_odoo_products" +#: ⭐ WAVE 28 (owner R1): *"ALL of Unique ID in Odoo is a database e.g. Customers/Products/Agents, +#: etc. Including expenses and GL codes."* Four more DOCUMENT/REGISTRY grains, each measured to +#: fit far inside `MAX_ROWS` (19 / 192 / 6,538 / 393 against 60,000). +AGENTS_KEY = "ut_odoo_agents" +ACCOUNTS_KEY = "ut_odoo_accounts" +BILLS_KEY = "ut_odoo_bills" +VENDORS_KEY = "ut_odoo_vendors" + +#: ⭐⭐ WAVE 30 / R7 / W30-T35 — THE TWO LINE GRAINS, AND THEY ARRIVE THE ONLY WAY THEY EVER COULD. +#: +#: ⚠ THE PARAGRAPH THAT STOOD HERE SAID THESE WERE "DELIBERATELY NOT HERE … at any cap", and it +#: was RIGHT ABOUT THE CAP AND WRONG ABOUT THE CONCLUSION — which is exactly why it is replaced +#: rather than left standing beside its own contradiction. The obstacle was never the number of +#: rows; it was that every row had to be COPIED into the shared `user_tables` document. MEASURED +#: on this box's mirror 2026-08-12: 254,189 order lines in the confirmed scope (256,810 unscoped) +#: and 963,783 GL lines — 4.2x and 16x `MAX_ROWS`, 63.9 MB and ~240 MB as JSON. Owner ruling R6 +#: settles what that means: *"there is no cap in how many data from the API source … can be pulled +#: into the app"*, so the answer is a different residency, never a bigger ceiling. +#: +#: ⛔ THESE TWO TABLES STORE NO ROWS HERE AND NEVER WILL. `routes_odoo_tables` binds them to the +#: DuckDB mirror (`GRID_SOURCES`) and `core.user_tables.row_limit` answers **0** for them — "this +#: database stores no rows HERE", which is a different statement from `None` ("connected and +#: uncapped") and from `MAX_ROWS` ("the editable substrate"). `plan()` below reads that evaluator +#: and builds no python row for either grain: 963,783 dicts in one process is the dangerous work +#: the answer exists to prevent. What DOES get written is the DEFINITION — a locked database with +#: fields, a label, grants and a nav entry, and zero rows. A definition with no rows is a working +#: grid; that is the whole shape of the conversion. +ORDER_LINES_KEY = "ut_odoo_order_lines" +GL_LINES_KEY = "ut_odoo_gl_lines" + +#: The join column the partner-grain tables carry. Derived links resolve through it (`on`/`from`). +JOIN_KEY = "partner_id" +#: The product-grain equivalent. +PRODUCT_JOIN_KEY = "product_id" +#: ⚠ AN AGENT IS A `res.partner`, so its id shares the partner namespace with a customer's — but +#: it is a DIFFERENT COLUMN on the customer row (`agent_id`, the customer's assigned agent) and the +#: two must never be joined through `JOIN_KEY`, which would link every customer to itself. +AGENT_JOIN_KEY = "agent_id" +#: A vendor is also a `res.partner`; same reasoning, its own column. +VENDOR_JOIN_KEY = "vendor_id" +ACCOUNT_JOIN_KEY = "account_code" + +#: The oracle's own predicate — `modules/ar._open_docs`, copied rather than re-derived so the two +#: cannot drift. It now selects a SUBSET of the invoices table rather than defining it. +_AR_OPEN = "payment_state IN ('not_paid','partial')" +_POSTED_DOCS = "state = 'posted' AND move_type IN ('out_invoice','out_refund')" +_AR_WHERE = f"{_POSTED_DOCS} AND {_AR_OPEN}" +_CONFIRMED = "state IN ('sale','done')" + +#: A refresh that would delete more than this share of a table's stored rows REFUSES instead. +#: ⛔ THE GUARD ONLY BECAME NECESSARY WHEN THE TABLES GOT BIG. `_ensure_table` removes rows that +#: left the population, which is right — a reversed invoice must not keep inflating a total. But +#: the population comes from the DuckDB mirror, and a mirror caught mid-resync (or one seeded +#: against an empty store) answers with FEWER rows and no error. At 1,228 rows that was a visible +#: mistake; at 31,418 it is a silent one. Odoo history does not halve, so a halving is a bad read. +MAX_SHRINK = 0.5 + + +def _ut(): + import core.user_tables as user_tables + return user_tables + + +def _registry(): + """`core.registry`, imported the same lazy way `_ut` is — this module is imported by the + route layer before `platform/` is necessarily on the path.""" + import core.registry as registry + return registry + + +def _iso_today(): + return _dt.date.today().strftime("%Y-%m-%d") + + +def _preset(field, flow="odoo_relational"): + """Stamp a field as machine-owned + preset — the `ut_ensure` lock_fields convention, so the + grid renders it grey and the preset walls refuse a rename or a delete.""" + field = dict(field) + field["automation"] = {"flowId": flow, "preset": True} + return field + + +#: The two conditions that reproduce `modules/ar`'s open-document predicate inside a rollup. +#: ⚠ Two `eq` legs joined by OR, not one `in` — `ROLLUP_CONDITION_OPS` has no `in`, and inventing +#: one here would be a second condition vocabulary beside `_clean_rollup`'s. +_OPEN_ONLY = {"conditions": [{"field": "payment_state", "op": "eq", "value": "not_paid"}, + {"field": "payment_state", "op": "eq", "value": "partial"}], + "conditionConj": "or"} + + +# --------------------------------------------------------------------------------------------- +# FIELD CONTRACTS +# --------------------------------------------------------------------------------------------- +# ⚠ Every type here must be in `core.user_tables.UT_FIELD_TYPES`, and `_clean_field` returns None +# for an unknown one — which DELETES the column silently on the next read rather than erroring. +def _scope_field(): + return {"key": "wholesale_scope", "label": "In wholesale scope", "type": "checkbox", + "source": "overlay", "default": False, + "description": "Unticked = the GIFTWARE DEALS / Amazon channel, which every wholesale " + "metric in this product excludes. The row is kept so no Odoo id is " + "missing; filter on this column to reconcile against the AR page."} + + +def _refreshed_field(): + return {"key": "refreshed", "label": "Refreshed", "type": "date", "source": "overlay", + "default": False, "description": "When this row was last reconciled against Odoo."} + + +def agent_fields(): + """One row per SALES AGENT, keyed on the `res.partner` id. + + ⭐ THE POPULATION IS A UNION OF TWO DISAGREEING SOURCES, and the disagreement is the reason it + is a union rather than a pick. MEASURED 2026-08-09: 16 partners carry commission lines, 17 + carry `res_partner.agent = TRUE`, and the union is 19 — so **2 agents earn commission without + the flag and 3 are flagged with no commission yet**. Either source alone silently drops real + agents. Same shape as `read_customers`' two document universes, for the same reason. + """ + return [_preset(f) for f in ( + {"key": "agent", "label": "Agent", "type": "text", "source": "overlay", + "default": True, "pinned": True}, + {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `res.partner` id. Also this row's id."}, + {"key": AGENT_JOIN_KEY, "label": "Odoo agent id", "type": "int", "source": "overlay", + "default": False}, + {"key": "flagged", "label": "Flagged in Odoo", "type": "checkbox", "source": "overlay", + "default": True, + "description": "Ticked = `res.partner.agent` is set. Unticked agents were found by " + "their commission lines instead - both are real, which is why this " + "table is the union of the two."}, + {"key": "commissioned", "label": "Has commission lines", "type": "checkbox", + "source": "overlay", "default": True}, + # ⭐ THE INVERSE HALF: the customers whose `agent_id` names this agent. MEASURED: 2,093 + # customers carry one and ALL 2,093 resolve to a row in this table (zero dangling). + # ⛔ W33-T43 / AMENDMENT A2 — the `customers` REVERSE link DELETED. See `invoice_fields` + # for the ruling. ⚠ This one costs more than the other two and the difference is worth + # recording: those were a click-through to a record whose id stays on the row, while this + # was an agent's BOOK — the list of customers assigned to them. The id side survives + # (`agent_id` here, and `agent_id` on `customer_data`), so the relationship is intact in + # the data and only the rendered list is gone; the same question is answerable on the + # customer grid by filtering `agent_id`, and at analytical grain via the `agent` dim on + # `sales_lines` / `sales_orders`. + _refreshed_field(), + )] + + +def account_fields(): + """One row per `account.account` — the GL chart, the owner's "GL codes". + + ⚠ NO LINK COLUMN, and that is a finding rather than an omission. A GL account meets the rest + of this schema only at LINE grain (963,783 `account_move_line` rows, 154,917 of them on + expense-type accounts), and a `ut_*` link folds rows that live in the store. The honest + binding is a read-through rollup naming a governed topic, or the mirror grid (R2) — never a + link into a table that does not exist. Declaring one here would render a permanently blank + column, which is the exact trap D-87 warns about from the value site. + """ + return [_preset(f) for f in ( + {"key": "account_code", "label": "Code", "type": "text", "source": "overlay", + "default": True, "pinned": True}, + {"key": "account_name", "label": "Account", "type": "text", "source": "overlay", + "default": True}, + {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `account.account` id. Also this row's id."}, + # ⚠ 15 DISTINCT VALUES MEASURED IN THE MIRROR, all declared. A `select` storing a value its + # options omit is wave-26 item 24: the filter panel answers with a list that cannot match + # what is stored. + {"key": "account_type", "label": "Type", "type": "select", "source": "overlay", + "default": True, + "options": ["expense", "expense_direct_cost", "expense_depreciation", "income", + "income_other", "asset_cash", "asset_current", "asset_receivable", + "asset_fixed", "asset_non_current", "asset_prepayments", + "liability_current", "liability_payable", "liability_credit_card", + "liability_non_current", "equity", "equity_unaffected", "off_balance"]}, + {"key": "is_expense", "label": "Expense account", "type": "checkbox", "source": "overlay", + "default": True, + "description": "Ticked for the expense family - the same predicate the semantic layer's " + "gl_lines topic uses, so this column and that topic cannot disagree."}, + _refreshed_field(), + )] + + +def vendor_fields(): + """One row per partner we have POSTED a vendor bill to, keyed on the `res.partner` id. + + ⚠ A VENDOR IS NOT A CUSTOMER TABLE ROW, even though both are `res.partner`. MEASURED: 393 + vendors, of which only 9 also appear in the customer population. Pointing bills at + `ut_odoo_customers` would have dangled 384 of 393 links — the failure would have been a mostly + empty column, not an error. + """ + return [_preset(f) for f in ( + {"key": "vendor", "label": "Vendor", "type": "text", "source": "overlay", + "default": True, "pinned": True}, + {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `res.partner` id. Also this row's id."}, + {"key": VENDOR_JOIN_KEY, "label": "Odoo vendor id", "type": "int", "source": "overlay", + "default": False}, + # ⭐⭐ W33-T48 / owner item 13 — the columns the census proved are there and were not shown. + # ⚠ `default` is deliberately split: the ones a vendor list is READ for (contact + tax id) + # arrive visible; the postal lines arrive HIDDEN, because four address columns turned on by + # default would push the useful ones off the first screen, and DESIGN.md's "never + # over-explain" applies to columns as much as to prose. Every one is one click away in the + # field picker, and a locked database still allows fields (item 4's vocabulary). + {"key": "country", "label": "Country", "type": "text", "source": "overlay", + "default": True}, + {"key": "email", "label": "Email", "type": "text", "source": "overlay", "default": True}, + {"key": "phone", "label": "Phone", "type": "text", "source": "overlay", "default": True}, + {"key": "mobile", "label": "Mobile", "type": "text", "source": "overlay", + "default": False}, + {"key": "vat", "label": "Tax ID", "type": "text", "source": "overlay", "default": True, + "description": "Odoo `vat` — the vendor's tax/VAT registration number."}, + # ⚠ `vendor_ref`, not `ref`: `ref` is Odoo's own name for it, and the bills grid already + # uses `ref` for the VENDOR'S INVOICE NUMBER on a document. Two different facts, and a + # shared spelling across two linked grids is how a rollup ends up summing the wrong column. + {"key": "vendor_ref", "label": "Vendor reference", "type": "text", "source": "overlay", + "default": False, + "description": "Odoo `res.partner.ref` — our internal reference for this vendor."}, + {"key": "website", "label": "Website", "type": "url", "source": "overlay", + "default": False}, + {"key": "street", "label": "Street", "type": "text", "source": "overlay", + "default": False}, + {"key": "street2", "label": "Street 2", "type": "text", "source": "overlay", + "default": False}, + {"key": "city", "label": "City", "type": "text", "source": "overlay", "default": False}, + {"key": "zip", "label": "ZIP", "type": "text", "source": "overlay", "default": False}, + {"key": "bills", "label": "Bills", "type": "link", "source": "overlay", "default": True, + "link": {"table": BILLS_KEY, "on": VENDOR_JOIN_KEY, "from": VENDOR_JOIN_KEY}}, + _refreshed_field(), + )] + + +def bill_fields(): + """One row per POSTED vendor bill or refund — the owner's "expenses", at DOCUMENT grain. + + ⚠ DOCUMENT GRAIN IS A CHOICE AND IT IS THE ONLY ONE THAT FITS: 6,538 bills against 154,917 + expense GL lines. What a person calls "expenses" is both, and they are different tables - the + bill is what you pay, the line is what it was coded to. This is the payable; the line ledger + is the read-through mirror grid (R2). + """ + return [_preset(f) for f in ( + {"key": "bill_no", "label": "Bill", "type": "text", "source": "overlay", + "default": True, "pinned": True}, + {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `account.move` id. Also this row's id."}, + {"key": "vendor", "label": "Vendor", "type": "text", "source": "overlay", "default": True}, + {"key": VENDOR_JOIN_KEY, "label": "Odoo vendor id", "type": "int", "source": "overlay", + "default": False}, + {"key": "invoice_date", "label": "Bill date", "type": "date", "source": "overlay", + "default": True}, + {"key": "due_date", "label": "Due date", "type": "date", "source": "overlay", + "default": True}, + # ⚠ SIGNED, like the customer side: Odoo's `_signed` fields already carry the refund's + # direction, so a refund reduces a total without anybody re-deriving a sign here. + {"key": "amount_untaxed", "label": "Billed $", "type": "currency", "source": "overlay", + "default": True, "agg": "sum"}, + {"key": "residual", "label": "Outstanding $", "type": "currency", "source": "overlay", + "default": True, "agg": "sum"}, + {"key": "payment_state", "label": "Payment state", "type": "select", "source": "overlay", + "default": True, + "options": ["not_paid", "partial", "in_payment", "paid", "reversed"]}, + {"key": "move_type", "label": "Document", "type": "select", "source": "overlay", + "default": False, "options": ["in_invoice", "in_refund"]}, + {"key": "vendor_link", "label": "Vendor record", "type": "link", "source": "overlay", + "default": False, + "link": {"table": VENDORS_KEY, "on": VENDOR_JOIN_KEY, "from": VENDOR_JOIN_KEY}}, + _refreshed_field(), + )] + + +def invoice_fields(): + """One row per POSTED customer invoice or refund — the full history, not just what is open.""" + return [_preset(f) for f in ( + {"key": "invoice_no", "label": "Invoice", "type": "text", "source": "overlay", + "default": True, "pinned": True}, + {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `account.move` id. Also this row's id."}, + {"key": "customer", "label": "Customer", "type": "text", "source": "overlay", + "default": True}, + {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay", + "default": False}, + {"key": "invoice_date", "label": "Invoice date", "type": "date", "source": "overlay", + "default": True}, + {"key": "due_date", "label": "Due date", "type": "date", "source": "overlay", + "default": True}, + {"key": "residual", "label": "Outstanding $", "type": "currency", "source": "overlay", + "default": True, "agg": "sum", + "description": "Odoo's signed residual. Exactly 0 on every settled document, which is " + "why AR rollups need no filter."}, + {"key": "amount_untaxed", "label": "Invoiced $", "type": "currency", "source": "overlay", + "default": True, "agg": "sum"}, + # ⛔ THE OPTION LIST WIDENED WITH THE POPULATION. It read ['not_paid','partial'] while the + # table held open AR only; the full posted history also carries paid / in_payment / + # reversed. A select holding a value its options do not declare is the wave-26 item-24 + # defect — the filter panel answers with a list that cannot match what is stored. + {"key": "payment_state", "label": "Payment state", "type": "select", "source": "overlay", + "default": True, + "options": ["not_paid", "partial", "in_payment", "paid", "reversed"]}, + {"key": "move_type", "label": "Document", "type": "select", "source": "overlay", + "default": False, "options": ["out_invoice", "out_refund"]}, + _scope_field(), + # ⭐ THE RECIPROCAL HALF (owner item 2, 2026-08-09). DERIVED (`on` declared), exactly like + # its twin, so the engine owns the cell and no human can edit a relation Odoo decided. + # ⛔⛔ W33-T43 / AMENDMENT A2 — `customer_link` DELETED, and the loss is stated here rather + # than left to be inferred from a green gate. + # + # It pointed at `ut_odoo_customers`, which W33-T44 retires (R2: one identity per subject). + # A2 ruled OPTION 2 — retire both twins, DROP the three link columns, keep every data + # column — so this grid loses the CLICK-THROUGH to a customer record and NOTHING else: + # `customer` (the name) and `partner_id` (the Odoo id) are plain columns on this same row, + # and `customer_data` now carries `partner_id` too, so both ends of the join still exist. + # + # ⛔ THE ALTERNATIVE WAS REFUSED IN WRITING, and the reason belongs beside the deletion: + # re-pointing this bag at `customer_data` needs `core/user_tables.py::_clean_link`'s `ut_` + # prefix test relaxed — which makes EVERY GATE GREEN while + # `automation_engine::compute_relation_cells` still resolves out of the `user_tables` + # document alone and returns `{}`. Empty cells, blank rollups, nothing red. A2 forbids + # touching that line this wave for exactly that reason. + # ⭐ DEBT D-88, closed 2026-08-09. `invoice_origin` carries the ORDER NAME an invoice was + # raised from, and the mirror did not sync it until this wave — so order->invoice was a + # two-hop join through 963,783 `account_move_line` rows and wave 27 shipped no link at all. + # ⚠ IT IS A NAME, NOT AN ID, and Odoo writes free text there (a manual invoice can hold + # anything; a merged one can hold several origins space-separated). The link resolves + # against `order_no` and finds nothing when the text is not an order name — the honest + # outcome, and the reason this is a join HINT rather than a foreign key. + {"key": "origin_order", "label": "Source order", "type": "text", "source": "overlay", + "default": False, + "description": "Odoo's `invoice_origin` - usually the order name, sometimes blank."}, + {"key": "order_link", "label": "Order record", "type": "link", "source": "overlay", + "default": False, + "link": {"table": ORDERS_KEY, "on": "order_no", "from": "origin_order"}}, + _refreshed_field(), + )] + + +def order_fields(): + """One row per CONFIRMED sale order — `state in (sale, done)`, the fixed wholesale scope.""" + return [_preset(f) for f in ( + {"key": "order_no", "label": "Order", "type": "text", "source": "overlay", + "default": True, "pinned": True}, + {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `sale.order` id. Also this row's id."}, + {"key": "customer", "label": "Customer", "type": "text", "source": "overlay", + "default": True}, + {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay", + "default": False}, + {"key": "order_date", "label": "Order date", "type": "date", "source": "overlay", + "default": True}, + {"key": "amount_untaxed", "label": "Order $", "type": "currency", "source": "overlay", + "default": True, "agg": "sum"}, + {"key": "team", "label": "Business unit", "type": "select", "source": "overlay", + "default": True, "options": ["Fisch", "Royal", "Sales", "Giftware Deals"]}, + {"key": "state", "label": "State", "type": "select", "source": "overlay", + "default": False, "options": ["sale", "done"]}, + {"key": "invoice_status", "label": "Invoice status", "type": "select", "source": "overlay", + "default": True, "options": ["invoiced", "to invoice", "upselling", "no"]}, + _scope_field(), + # ⛔ W33-T43 / AMENDMENT A2 — `customer_link` DELETED here too; see the note on the same + # column in `invoice_fields` above for the ruling and the refused alternative. This row + # keeps `customer` and `partner_id`, so only the click-through is gone. + # The reciprocal of the invoice's `order_link` (D-88): the invoices raised from THIS + # order, matched on the order's own name. + {"key": "invoices", "label": "Invoices", "type": "link", "source": "overlay", + "default": True, + "link": {"table": INVOICES_KEY, "on": "origin_order", "from": "order_no"}}, + _refreshed_field(), + )] + + +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# THE TWO READ-THROUGH GRAINS (W30-T35). Their rows are SERVED FROM THE MIRROR, never stored. +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⛔⛔ THE KEY SET IS HALF OF A CONTRACT AND `routes_odoo_tables.GRID_SOURCES[…]["cols"]` IS THE +# OTHER HALF. It binds a key to SQL; the label, type and order are declared HERE, once. The two +# lists must name the SAME columns, and both failure directions are silent: +# * a bound key with no declaration -> a silent NO-CELL (the row projection is strict); +# * a declared key with no binding -> an INACTIVE filter leaf, which WIDENS the result set. +# `verify_scopes.section_line_grids` compares the two sets, which is why neither side may "just +# add a column". +# +# ⛔ NO LINK COLUMN AND NO `refreshed` STAMP ON EITHER TABLE, and both absences are findings +# rather than omissions: +# * a LINK folds the rows the TARGET table STORES (`compute_relation_cells` reads the raw +# document, not the mirror), so a link at a read-through grain resolves against nothing and +# renders a permanently blank column — the trap `account_fields` names from the other end. +# The join ids ride as ordinary filterable columns instead, so the relationships are all +# still reachable by a person and by SQL. +# * `refreshed` means "when this row was last reconciled against Odoo", and it is stamped by +# `_ensure_table_inplace` onto rows it WRITES. Nothing here is ever written, so the column +# would be blank for every row forever. `section_line_grids` tolerates the key; the honest +# thing is not to declare it. + + +def order_line_fields(): + """One row per `sale.order.line` on a CONFIRMED order — the same `state in (sale, done)` + scope every wholesale metric in this product uses. + + MEASURED on the mirror 2026-08-12: **254,189 lines in scope** of 256,810 (the 2,621 excluded + sit on draft/sent/cancelled orders). Every line in scope is on a `sale` order — zero `done` — + but `done` stays in the option list because it is in the SCOPE, and a filter offering only + what happens to be stored today goes stale the first time an order is marked done. + + ⛔ THE SCOPE, THE ORDER DATE AND THE ORDER NAME ALL LIVE ACROSS A JOIN. `sale_order_line` + carries no `state` at all (12 columns, measured), so the binding is a join to `sale_order` — + which is also what makes `order_no` a readable primary cell instead of a line id. + + ⚠ `qty` IS `int`, NOT `currency`, AND THAT IS A MEASUREMENT. 3,888 of 256,810 lines carry a + FRACTIONAL quantity (0.2, 0.4, 0.5, 1.66 …) and the minimum is -1.0, so the question "does the + type truncate?" had to be answered rather than assumed: it does not. `int` and `currency` both + render through the client's `numberText`, which rounds nothing without a `format.decimals` + bag — the only difference is the `$` a `currency` column prepends. A quantity is not money, so + it takes the type that does not paint one. + """ + return [_preset(f) for f in ( + {"key": "order_no", "label": "Order", "type": "text", "source": "overlay", + "default": True, "pinned": True, + "description": "The sale order this line belongs to. Zero orders have a blank name, " + "which is why it is the primary cell rather than the line id."}, + {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `sale.order.line` id. Also this row's id."}, + {"key": "order_id", "label": "Odoo order id", "type": "int", "source": "overlay", + "default": False, + "description": "The `sale.order` id — the key `ut_odoo_orders` is keyed on."}, + {"key": "customer", "label": "Customer", "type": "text", "source": "overlay", + "default": True}, + {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay", + "default": False}, + {"key": "product", "label": "Product", "type": "text", "source": "overlay", + "default": True, + "description": "Blank on the 101 section and note lines, which carry no product."}, + {"key": PRODUCT_JOIN_KEY, "label": "Odoo product id", "type": "int", "source": "overlay", + "default": False}, + {"key": "qty", "label": "Qty", "type": "int", "source": "overlay", "default": True, + "agg": "sum", + "description": "Ordered quantity. 3,888 lines carry a fraction and some are negative " + "(returns), so nothing here is rounded."}, + {"key": "price_subtotal", "label": "Line $", "type": "currency", "source": "overlay", + "default": True, "agg": "sum"}, + {"key": "margin", "label": "Margin $", "type": "currency", "source": "overlay", + "default": False, "agg": "sum", + "description": "Odoo's own line margin. Populated on every line."}, + {"key": "purchase_price", "label": "Unit cost", "type": "currency", "source": "overlay", + "default": False, + "description": "The cost Odoo priced this line's margin against, per unit."}, + {"key": "order_date", "label": "Order date", "type": "date", "source": "overlay", + "default": True}, + {"key": "state", "label": "State", "type": "select", "source": "overlay", + "default": False, "options": ["sale", "done"]}, + _scope_field(), + )] + + +def gl_line_fields(): + """One row per `account.move.line` — the general ledger, and the owner's "expenses" at the + grain a person can actually browse. + + MEASURED 2026-08-12: **963,783 lines**, of which 944,846 posted, 18,885 cancelled and 52 + draft. ⛔ UNSCOPED ON PURPOSE — a general ledger whose draft and cancelled entries are + invisible is a ledger that cannot be reconciled, so `parent_state` rides as a COLUMN and the + reader chooses. That is the same decision the binding states from the SQL side. + + ⚠ TWO COLUMNS ARE LEGITIMATELY BLANK ON REAL ROWS, named here so neither reads as a defect: + **61,911 lines carry no partner** (journal entries that are not about a customer), and + **1,727 carry no account** at all, which is also why `account_code` — the key + `ut_odoo_accounts` is keyed on — is blank on exactly those 1,727 and the join that supplies + it is a LEFT one. + + ⚠ `move_type` IS `text`, NOT `select`, and it is the `product_fields.category` argument: + five values exist today (`out_invoice` 515,634 · `entry` 414,992 · `in_invoice` 18,901 · + `out_refund` 14,061 · `in_refund` 195) and Odoo's enum is longer than what we happen to hold. + A select whose options go stale answers a filter with a list that cannot match a stored value + (wave-26 item 24). `line_type` and `parent_state` ARE selects because their option lists were + measured COMPLETE against the whole table. + """ + return [_preset(f) for f in ( + {"key": "entry", "label": "Entry", "type": "text", "source": "overlay", + "default": True, "pinned": True, + "description": "The journal entry this line belongs to. Never blank."}, + {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `account.move.line` id. Also this row's id."}, + {"key": "move_id", "label": "Odoo entry id", "type": "int", "source": "overlay", + "default": False}, + {"key": "account", "label": "Account", "type": "text", "source": "overlay", + "default": True}, + {"key": ACCOUNT_JOIN_KEY, "label": "Account code", "type": "text", "source": "overlay", + "default": True, + "description": "The GL code, from the joined chart of accounts — the key " + "`ut_odoo_accounts` is keyed on. Blank on the 1,727 lines with no " + "account."}, + {"key": "customer", "label": "Partner", "type": "text", "source": "overlay", + "default": True, + "description": "Blank on the 61,911 lines that are not about a partner."}, + {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay", + "default": False}, + {"key": "date", "label": "Date", "type": "date", "source": "overlay", "default": True}, + {"key": "debit", "label": "Debit", "type": "currency", "source": "overlay", + "default": True, "agg": "sum"}, + {"key": "credit", "label": "Credit", "type": "currency", "source": "overlay", + "default": True, "agg": "sum"}, + {"key": "balance", "label": "Balance", "type": "currency", "source": "overlay", + "default": True, "agg": "sum", + "description": "Debit minus credit, as Odoo stores it. Sums to zero over a whole entry."}, + {"key": "line_type", "label": "Line type", "type": "select", "source": "overlay", + "default": False, + "options": ["product", "cogs", "payment_term", "line_note", "line_section"]}, + {"key": "move_type", "label": "Document type", "type": "text", "source": "overlay", + "default": False}, + {"key": "parent_state", "label": "Entry state", "type": "select", "source": "overlay", + "default": True, "options": ["draft", "posted", "cancel"]}, + _scope_field(), + )] + + +def product_fields(): + """One row per `product.product`, keyed on its id — EVERY product, archived ones included. + + ⚠ THE ROW ID IS THE PRODUCT ID, NOT THE SKU CODE, and the difference is measurable: 12 codes + map to more than one product id (re-SKU / merge history). The code is what a human reads and + the id is what `sales_lines.product` groups by, so both are columns and only the id is the + identity. + + ⛔ THE PINNED COLUMN IS THE NAME, NOT THE SKU, and that is not a style choice. 62 products + carry no `default_code` at all (UBER CHARGE, Delivery Charges, PICK UP …) while ZERO carry a + blank name — measured. Pinning `code` would give those rows a blank primary cell, which is + exactly D-80: a first column nothing populates quietly becoming the row's identity + ([[fallback-that-became-the-rule]]). + """ + return [_preset(f) for f in ( + {"key": "product", "label": "Product", "type": "text", "source": "overlay", + "default": True, "pinned": True}, + {"key": PRODUCT_JOIN_KEY, "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `product.product` id. Also this row's id, and what " + "every product rollup groups by."}, + {"key": "code", "label": "SKU", "type": "text", "source": "overlay", + "default": True, "description": "Odoo's `default_code`. Blank on the 62 charge/service " + "products that are not stocked SKUs."}, + {"key": "active", "label": "Active in Odoo", "type": "checkbox", "source": "overlay", + "default": True, + "description": "Unticked = archived. Archived products are kept because they still " + "carry sales history — two of them sold this year."}, + # ⚠ TEXT, NOT SELECT. 71 categories exist today and Odoo gains them without telling us; a + # select whose options go stale answers a filter with a list that cannot match a stored + # value (wave-26 item 24). Text filters honestly and never goes out of date. + {"key": "category", "label": "Category", "type": "text", "source": "overlay", + "default": True}, + {"key": "product_type", "label": "Type", "type": "select", "source": "overlay", + "default": False, "options": ["product", "consu", "service"]}, + {"key": "standard_price", "label": "Standard cost", "type": "currency", + "source": "overlay", "default": True}, + # ⭐ SOURCE-BACKED (read-through) — the product-grain half of "compute all of the data in + # Odoo". It names a governed TOPIC + METRIC KEY and one grouped query answers every SKU; + # `sales_lines` holds 256,810 rows that are never copied into this table. + {"key": "sales_ytd", "label": "Sales YTD", "type": "rollup", "source": "overlay", + "default": True, "agg": "sum", + "rollup": {"source": {"topic": "sales_lines", "measure": "revenue", + "groupBy": "product", "on": PRODUCT_JOIN_KEY, "window": "ytd"}}}, + {"key": "units_ytd", "label": "Units YTD", "type": "rollup", "source": "overlay", + "default": True, "agg": "sum", + "rollup": {"source": {"topic": "sales_lines", "measure": "units", + "groupBy": "product", "on": PRODUCT_JOIN_KEY, "window": "ytd"}}}, + {"key": "margin_ytd", "label": "Gross margin YTD $", "type": "rollup", "source": "overlay", + "default": True, "agg": "sum", + "rollup": {"source": {"topic": "sales_lines", "measure": "margin", + "groupBy": "product", "on": PRODUCT_JOIN_KEY, "window": "ytd"}}}, + _refreshed_field(), + )] + + +def customer_fields(): + """One row per partner Odoo has transacted with, keyed on the `res.partner` id. + + ⭐ Two DERIVED links (`on` declared), so the engine owns both cells and a human cannot edit a + relation Odoo already decided. The rollups come in two kinds on purpose: + * LINK rollups fold the rows in `ut_odoo_invoices` / `ut_odoo_orders` — they can answer + anything about a document the table holds, including a date rank; + * SOURCE rollups name a governed topic + metric and are answered by ONE grouped SQL query + over the whole mirror — they can answer a DATE-WINDOWED money question, which a link + rollup cannot, because a condition can only compare against a literal and a literal year + start is right until 1 January. + """ + return [_preset(f) for f in ( + {"key": "customer", "label": "Customer", "type": "text", "source": "overlay", + "default": True, "pinned": True}, + {"key": JOIN_KEY, "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `res.partner` id. Also this row's id."}, + # ⭐⭐ W37-T11 / D-165 — THE POSTAL ADDRESS, which this table declared nowhere and served + # never. `verify_odoo_relational._prove_customer_address` recorded the blocker in its own + # docstring on 2026-08-12 — *"`street`/`street2`/`zip` are not among [the mirror's 16 + # columns] … blocked on a `sync_all()` backfill no ticket in wave 30 owns"*. That backfill + # HAS since run (`_sync_state` carries `res_partner.cols.…,street,street2,…,zip`, done + # 2026-08-15), so the half that could not be built now can be. + # ⛔ THIS IS ALSO LANE D'S DEPENDENCY. R4 puts address→coordinates in an enrichment field, + # and a geocoder pointed at a column that does not exist enriches nothing. + # ⚠ `default: False` on the postal lines, `True` on city/state — the same split + # `vendor_fields` made and for the same reason: four address columns on by default push + # the useful ones off the first screen. A locked database still allows fields, so every + # one is a click away in the picker. + {"key": "street", "label": "Street", "type": "text", "source": "overlay", + "default": False}, + {"key": "street2", "label": "Street 2", "type": "text", "source": "overlay", + "default": False}, + {"key": "city", "label": "City", "type": "text", "source": "overlay", "default": True}, + {"key": "state", "label": "State", "type": "text", "source": "overlay", "default": True}, + {"key": "zip", "label": "ZIP", "type": "text", "source": "overlay", "default": False}, + {"key": "country", "label": "Country", "type": "text", "source": "overlay", + "default": False}, + {"key": "agent", "label": "Sales agent", "type": "text", "source": "overlay", + "default": True, + "description": "The customer's assigned agent (res.partner.agent_ids[0] — the " + "Customers-module convention)."}, + # ⭐ WAVE 28 — the agent's ID beside its NAME, because a link joins on an id and this + # table carried only the display string. ⚠ It is `agent_id`, NEVER `partner_id`: both are + # `res.partner` ids, and joining agents through `JOIN_KEY` would link every customer to + # itself and look plausible doing it. + {"key": AGENT_JOIN_KEY, "label": "Odoo agent id", "type": "int", "source": "overlay", + "default": False}, + _scope_field(), + + # --- the relations ------------------------------------------------------------------- + {"key": "invoices", "label": "Invoices", "type": "link", "source": "overlay", + "default": True, + "link": {"table": INVOICES_KEY, "on": JOIN_KEY, "from": JOIN_KEY}}, + {"key": "orders", "label": "Orders", "type": "link", "source": "overlay", + "default": True, + "link": {"table": ORDERS_KEY, "on": JOIN_KEY, "from": JOIN_KEY}}, + # MEASURED: 2,093 customers carry an `agent_id` and all 2,093 resolve to a row in the + # agents table — zero dangling, which is why this ships as a link rather than a lookup. + {"key": "agent_link", "label": "Agent record", "type": "link", "source": "overlay", + "default": False, + "link": {"table": AGENTS_KEY, "on": AGENT_JOIN_KEY, "from": AGENT_JOIN_KEY}}, + + # --- link rollups over the invoice history -------------------------------------------- + # ⭐ NO CONDITION, and that is measured rather than assumed: a settled document's residual + # is exactly 0, so summing the full history gives the open balance to the cent. + # ⚠ THE LABEL SAYS "ALL CHANNELS" BECAUSE THE COLUMN TOTAL DOES NOT MATCH THE AR PAGE. + # Per customer this is exactly right. Summed down the column it is $2,347,608.49 while + # `Settings → AR` shows $591,828.54 — a 4x gap that is entirely the GIFTWARE DEALS / + # Amazon partner, which wholesale scope excludes and this table deliberately keeps. Two + # numbers with one name, 4x apart, in one product is how a correct figure gets reported + # as a bug; the scope belongs in the label, not only in a column somebody has to filter. + {"key": "ar_outstanding", "label": "AR outstanding $ - all channels", "type": "rollup", + "source": "overlay", "default": True, "agg": "sum", + "description": "Open balance across every posted document, INCLUDING the Amazon " + "channel. Filter `In wholesale scope` to reconcile with the AR page.", + "rollup": {"link": "invoices", "field": "residual", "fn": "sum"}}, + {"key": "invoiced_all_time", "label": "Invoiced $ - all time", "type": "rollup", + "source": "overlay", "default": False, "agg": "sum", + "rollup": {"link": "invoices", "field": "amount_untaxed", "fn": "sum"}}, + # ⛔ THESE TWO DO NEED THE PREDICATE. Over the widened link a bare `countall` counts every + # document ever posted and labels it "open invoices" — the wrong-number-that-looks-right + # this module refuses everywhere else. + {"key": "open_invoices", "label": "Open invoices #", "type": "rollup", + "source": "overlay", "default": True, + "rollup": {"link": "invoices", "fn": "countall", **_OPEN_ONLY}}, + # ⛔ NOT `min`. `_rollup_fold`'s min/max are NUMERIC folds (`_lane_num`), so `min` over a + # date column finds no numbers and returns BLANK — a column that renders empty forever + # while looking configured. Ranking a DATE is what `latest` + `sortBy` is for. + {"key": "oldest_due", "label": "Oldest due date", "type": "rollup", "source": "overlay", + "default": True, + "rollup": {"link": "invoices", "field": "due_date", "fn": "latest", + "sortBy": "due_date", "sortDir": "asc", **_OPEN_ONLY}}, + + # --- link rollups over the order history ---------------------------------------------- + {"key": "order_count", "label": "Orders #", "type": "rollup", "source": "overlay", + "default": True, + "rollup": {"link": "orders", "fn": "countall"}}, + {"key": "last_order", "label": "Last order date", "type": "rollup", "source": "overlay", + "default": True, + "rollup": {"link": "orders", "field": "order_date", "fn": "latest", + "sortBy": "order_date", "sortDir": "desc"}}, + + # --- source-backed (read-through) rollups --------------------------------------------- + # ⛔ THESE DO NOT AND CANNOT COME FROM THE `invoices` LINK. `ut_odoo_invoices` is posted + # BILLING; `revenue_invoiced` is ORDER-LINE revenue narrowed by the order's fully-invoiced + # flag. Different grain, different question — the metric KEY carries the distinction, + # which is the whole reason a rollup may not carry SQL of its own. + {"key": "sales_ytd", "label": "Sales YTD - invoiced", "type": "rollup", + "source": "overlay", "default": True, "agg": "sum", + "rollup": {"source": {"topic": "sales_lines", "measure": "revenue_invoiced", + "groupBy": "order_partner", "on": JOIN_KEY, "window": "ytd"}}}, + {"key": "sales_ltm", "label": "Sales LTM", "type": "rollup", "source": "overlay", + "default": True, "agg": "sum", + "rollup": {"source": {"topic": "sales_lines", "measure": "revenue", + "groupBy": "order_partner", "on": JOIN_KEY, "window": "ltm"}}}, + {"key": "margin_ytd", "label": "Gross margin YTD $", "type": "rollup", "source": "overlay", + "default": False, "agg": "sum", + "rollup": {"source": {"topic": "sales_lines", "measure": "margin", + "groupBy": "order_partner", "on": JOIN_KEY, "window": "ytd"}}}, + {"key": "orders_ytd", "label": "Orders YTD #", "type": "rollup", "source": "overlay", + "default": False, + "rollup": {"source": {"topic": "sales_orders", "measure": "orders", + "groupBy": "partner", "on": JOIN_KEY, "window": "ytd"}}}, + _refreshed_field(), + )] + + +# --------------------------------------------------------------------------------------------- +# READING THE MIRROR +# --------------------------------------------------------------------------------------------- +def excluded_names(): + """The partner names out of wholesale scope, from the ONE place that defines them. + + Read through `core.odoo` rather than re-listed here: a second literal is a second scope, and + the day somebody adds a channel this module would keep answering the old question. + """ + try: + import core.odoo as odoo + names = getattr(odoo, "EXCLUDE_PARTNER_NAMES", None) or set() + return {str(n).strip().lower() for n in names if str(n).strip()} + except Exception: # noqa: BLE001 + return set() + + +def excluded_ids(cur, names=None): + """The out-of-scope partner IDS, resolved against the MIRROR. + + ⭐ IDS, NOT THE DENORMALISED NAME ON THE DOCUMENT, for two reasons that both bite. + `account_move.partner_name` is a copy taken when the document was written, and this module's + own `customers_from` says so out loud — *"a partner's name can differ across documents + (renames land on new invoices only)"*. So a rename would put some of one partner's documents + in scope and the rest out, silently, and the totals would stop reconciling with nothing to + point at. `modules/ar`, the oracle these numbers answer to, has always excluded by ID. + + ⛔ RESOLVED FROM THE MIRROR, NOT `core.odoo.excluded_partner_ids()`. That function issues a + LIVE `search_read`, so importing it here would make spawning four locked databases fail + whenever Odoo is unreachable — including on this developer machine, where the handshake dies + on an expired certificate ([[local-odoo-ssl-quirk]]). Same names, same answer, no network. + + ⚠ MATCHED CASE- AND WHITESPACE-INSENSITIVELY, and a NULL name simply does not match — which + is the correct direction. 44 transacting partners carry no name at all; treating an + unanswerable name as "excluded" would drop $7,734.83 of real open AR out of scope. + """ + names = names if names is not None else excluded_names() + if not names: + return set() + rows = cur.execute("SELECT id, name FROM res_partner WHERE name IS NOT NULL").fetchall() + return {int(pid) for pid, name in rows if str(name).strip().lower() in names} + + +def columns(cur, table): + """The column names a mirror table actually has, lowercased. `set()` if the table is absent. + + ⛔ WHY THIS EXISTS, AND IT COST A LIVE 500. `harness.datastore.ready()` gates on ENTITY + phases, and a Space hydrates its mirror from `store_seed/royal.duckdb` — a SNAPSHOT. Columns + added to `ENTITIES` after that snapshot was taken (`res_partner.agent_id`, + `account_move_line.product_id`, …) are backfilled by `sync_all()` under their OWN `_sync_state` + keys, which `ready()` does not read. So there is a real window, right after a boot, where the + store reports READY and a column this module names does not exist yet — and DuckDB answers a + missing identifier with a Binder error, which reached the operator as a bare `500`. + ⚠ The absent columns are all DISPLAY ones (an agent name, a category, a team). Refusing the + whole spawn over a cosmetic column would be worse than the gap it is reporting, so the readers + degrade the COLUMN to blank and still write every id. + """ + try: + rows = cur.execute(f"SELECT * FROM {table} LIMIT 0") + return {str(d[0]).lower() for d in rows.description} + except Exception: # noqa: BLE001 + return set() + + +def _col(have, name, default="NULL"): + """`name` when the mirror has it, else a literal that keeps the SELECT's arity intact.""" + return name if str(name).split(".")[-1].lower() in have else default + + +def _as_date(value): + """ISO date string, or ''. The grid renders `date` cells itself (W26: `Aug 5, 2026`), so the + STORED value stays ISO — a formatted string in the cell is a value the filters cannot sort.""" + if not value: + return "" + return str(value)[:10] + + +def _in_scope(pid, excluded): + """`'1'` | `''` — the `checkbox` cell convention (`aios_grid`: the overlay stores '1' or '').""" + return "" if int(pid) in excluded else "1" + + +def read_invoices(cur, excluded=None, open_only=False): + """[(row dict)] — posted customer invoices and refunds, keyed on the `account.move` id. + + ONE reader, two projections. `open_only` applies the AR oracle's predicate and drops the + out-of-scope channel, which is what `read_open_ar` wants; the default keeps every row and + TAGS the channel instead. Two queries would be two populations, and they drift the moment + either is edited. + + Takes a CURSOR so a gate can hand it a fixture connection; no global store binding here. + """ + excluded = excluded if excluded is not None else excluded_ids(cur) + where = _AR_WHERE if open_only else _POSTED_DOCS + # ⚠ `invoice_origin` (D-88) is read through `_col` DELIBERATELY. It was added to `ENTITIES` in + # this same wave, so a Space whose mirror is still hydrating from a pre-wave seed snapshot does + # not have the column yet — and DuckDB answers a missing identifier with a Binder error that + # reaches the operator as a bare 500. This is the exact class `columns()` was written for: the + # link degrades to blank for one sync cycle instead of refusing the whole spawn. + have = columns(cur, "account_move") + sql = ("SELECT id, name, partner_id, partner_name, invoice_date, invoice_date_due, " + " amount_untaxed_signed, amount_residual_signed, payment_state, move_type, " + f" {_col(have, 'invoice_origin', chr(39) + chr(39))} " + f"FROM account_move WHERE {where} AND partner_id IS NOT NULL") + out = [] + for r in cur.execute(sql).fetchall(): + (mid, name, pid, pname, inv_date, due, untaxed, residual, pay_state, mtype, origin) = r + scope = _in_scope(pid, excluded) + if open_only and not scope: + continue + out.append({ + "_id": str(mid), + "invoice_no": str(name or ""), + "odoo_id": int(mid), + "customer": str(pname or ""), + JOIN_KEY: int(pid), + "invoice_date": _as_date(inv_date), + "due_date": _as_date(due), + "residual": float(residual or 0.0), + "amount_untaxed": float(untaxed or 0.0), + "payment_state": str(pay_state or ""), + "move_type": str(mtype or ""), + "origin_order": str(origin or "").strip(), + "wholesale_scope": scope, + }) + return out + + +def read_open_ar(cur, excluded=None): + """The OPEN, wholesale-scoped subset — `modules/ar._open_docs`' own population. + + Kept as its own door because `modules/ar` is this module's oracle for the AR numbers, and an + oracle answers exactly one question. It is a projection of `read_invoices`, never a second + query. + """ + return read_invoices(cur, excluded=excluded, open_only=True) + + +def read_orders(cur, excluded=None): + """[(row dict)] — confirmed sale orders, keyed on the `sale.order` id.""" + excluded = excluded if excluded is not None else excluded_ids(cur) + have = columns(cur, "sale_order") + sql = (f"SELECT id, name, date_order, partner_id, partner_name, {_col(have, 'team_name')}, " + f" state, amount_untaxed, {_col(have, 'invoice_status')} " + f"FROM sale_order WHERE {_CONFIRMED} AND partner_id IS NOT NULL") + out = [] + for r in cur.execute(sql).fetchall(): + (oid, name, when, pid, pname, team, state, untaxed, inv_status) = r + out.append({ + "_id": str(oid), + "order_no": str(name or ""), + "odoo_id": int(oid), + "customer": str(pname or ""), + JOIN_KEY: int(pid), + "order_date": _as_date(when), + "amount_untaxed": float(untaxed or 0.0), + "team": str(team or ""), + "state": str(state or ""), + "invoice_status": str(inv_status or ""), + "wholesale_scope": _in_scope(pid, excluded), + }) + return out + + +def read_products(cur): + """[(row dict)] — EVERY `product.product`, keyed on its id. + + ⛔ NO `active` AND NO `default_code` FILTER, and both exclusions were measured before they + were dropped. Filtering to active-and-coded gave 5,829 of 5,948 rows and left FIVE products + that sold this very year with no row at all: two archived SKUs (`9SAT-FY`, `2GSTY`) and three + uncoded charge lines (`UBER CHARGE`, `[Delivery_009] Delivery Charges`, `PICK UP`). A product + grouped by `sales_lines.product` that has no parent row is a rollup value with nowhere to + land — silently. 119 extra rows is the whole cost of the claim being literally true. + """ + have = columns(cur, "product_product") + sql = (f"SELECT id, default_code, name, {_col(have, 'categ_name')}, type, " + f" {_col(have, 'standard_price', '0')}, {_col(have, 'active', 'TRUE')} " + "FROM product_product") + out = [] + for r in cur.execute(sql).fetchall(): + (prid, code, name, categ, ptype, cost, active) = r + out.append({ + "_id": str(prid), + "product": str(name or ""), + PRODUCT_JOIN_KEY: int(prid), + "code": str(code or ""), + "active": "1" if active else "", + "category": str(categ or ""), + "product_type": str(ptype or ""), + "standard_price": float(cost or 0.0), + }) + return out + + +def read_customers(cur, excluded=None): + """[(row dict)] — every CUSTOMER partner, keyed on the `res.partner` id. + + ⭐ THE POPULATION IS A UNION OF THREE LEGS, and every one of them is load-bearing. + + The two DOCUMENT legs are the original pair: Amazon books as direct invoices with no sale + order (the `odoo-api` gotcha), so a sale-order leg alone would silently drop a real customer. + + ⭐⭐ THE THIRD IS `customer_rank > 0 AND active` (wave 29, item 22 / R12 via finding F2 — + the owner's *"never an arbitrary limit… applies to ALL connected database"*). The old + docstring said partners with no document are *"left out on purpose — a row that can never + appear in any topic has nothing to roll up"*; that reasoning is RETIRED. It is the same + join-drop class as the Product grid's 2,717, and it dropped **~1,149 real customer records** + (MEASURED 2026-08-11: `rank>0 active` = 3,617 against a document union of ~2,000). A customer + a salesperson has not sold to yet is exactly the row a prospecting view needs. + + ⛔ IT IS A UNION AND NOT A REPLACEMENT, AND THAT IS MEASURED, NOT TIDINESS. Swapping the + document legs for the rank leg would drop **16 partners that hold posted documents** (7 + archived, 9 active with rank <= 0), and `ut_odoo_invoices` / `ut_odoo_orders` rows carry + `partner_id` LINKS straight back here — so those links would dangle with nothing reporting + it. When other tables point AT a population, a widening must be a SUPERSET. + + ⚠ THE RANK LEG IS SKIPPED WHEN THE MIRROR HAS NO `customer_rank` COLUMN, which is the same + `columns()`/`_col` discipline every other optional column here uses — but note the difference + honestly: an absent `agent_id` blanks a CELL, while an absent `customer_rank` narrows the + POPULATION back to the document union. It degrades to today's behaviour rather than to an + empty or a wrong table, and `verify_odoo_relational` carries a check that goes RED while the + column is missing so the narrowing can never pass for done. + + ⛔ NOT FROM LIVE ODOO, THOUGH `customer_rank` IS TRIVIAL TO ASK IT. `excluded_ids` above + states the rule for this module and it applies with more force to a POPULATION than to a name + list: a live call makes the spawn fail whenever Odoo is unreachable, and a Space hydrates its + mirror from a SNAPSHOT at boot. The population would then be "whichever source answered this + time" — swinging ~45% against `MAX_SHRINK`'s 50% refusal, deleting and re-adding rows on the + weather. One source, always present at spawn time: the mirror. + + ⛔ NOT DERIVED FROM THE INVOICE ROWS. `customers_from` did that when the table WAS the open-AR + partners; sourcing a customer registry from its own receivables is what kept most Odoo ids + out of the store in the first place. + """ + excluded = excluded if excluded is not None else excluded_ids(cur) + have = columns(cur, "res_partner") + # ⚠ THE AGENT JOIN IS DROPPED WHOLE when `agent_id` is absent, not merely NULL-ed: the join + # itself names the column, so `_col` on the SELECT list alone would still fail to bind. + agent = ("ag.name" if "agent_id" in have else "NULL") + agent_id_col = ("p.agent_id" if "agent_id" in have else "NULL") + join = ("LEFT JOIN res_partner ag ON ag.id = p.agent_id " if "agent_id" in have else "") + # ⚠ BOTH columns must be present, not just `customer_rank`: `active` is what keeps an + # archived prospect out, and a rank test without it would re-admit the 47 archived partners + # the mirror carries. Absent ⇒ the leg is dropped WHOLE, exactly like the agent join above. + rank_leg = (" OR (p.customer_rank > 0 AND p.active) " + if {"customer_rank", "active"} <= have else "") + # ⭐⭐ W37-T11 / D-165 — the POSTAL columns join city/state/country here. They ride `_col` like + # every other optional column, so a mirror predating the 2026-08-15 backfill degrades the CELL + # to blank rather than failing the spawn with a Binder error (see `columns()`'s header for the + # live 500 that rule was written after). + sql = (f"SELECT p.id, p.name, {_col(have, 'p.city')}, {_col(have, 'p.state_name')}, " + f" {_col(have, 'p.country_name')}, {agent}, {agent_id_col}, " + f" {_col(have, 'p.street')}, {_col(have, 'p.street2')}, {_col(have, 'p.zip')} " + "FROM res_partner p " + f"{join}" + "WHERE p.id IN (" + f" SELECT partner_id FROM sale_order WHERE {_CONFIRMED} AND partner_id IS NOT NULL " + " UNION " + " SELECT partner_id FROM account_move " + f" WHERE {_POSTED_DOCS} AND partner_id IS NOT NULL)" + f"{rank_leg}") + out = [] + for r in cur.execute(sql).fetchall(): + (pid, name, city, state, country, agent, agent_id, street, street2, zipc) = r + out.append({ + "_id": str(pid), + "customer": str(name or ""), + JOIN_KEY: int(pid), + "street": str(street or ""), + "street2": str(street2 or ""), + "city": str(city or ""), + "state": str(state or ""), + "zip": str(zipc or ""), + "country": str(country or ""), + "agent": str(agent or ""), + AGENT_JOIN_KEY: int(agent_id) if agent_id else "", + "wholesale_scope": _in_scope(pid, excluded), + }) + return out + + +def read_agents(cur): + """[(row dict)] — the UNION of both agent sources, keyed on the `res.partner` id. + + ⛔ `res_partner.agent` is a BOOLEAN and `datastore.BOOL_FIELDS` lists it for a measured reason: + Odoo returns False both for "empty" and for "boolean false", so a bool missing from that list + silently becomes NULL and every row would read "not an agent" indistinguishably from + "unknown". Read it as a truth value, never as a presence test. + """ + have = columns(cur, "res_partner") + if "id" not in have: + return [] + flagged = "p.agent" if "agent" in have else "FALSE" + # ⛔⛔ THE COMMISSION TABLE IS GUARDED AS A **TABLE**, not just as a column, and that + # distinction is the whole point of this block. `columns()` was written for a missing COLUMN + # (a backfill that has not run yet); `account_invoice_line_agent` is an OCA module entity that + # a mirror hydrated from an older seed snapshot may not have AT ALL. A SELECT naming an absent + # table is a DuckDB Binder error, and this reader runs inside `plan()` — so one missing table + # would fail the WHOLE eight-table spawn and reach the operator as a bare 500. That is + # precisely D-107's shape, and it would have arrived on the first deploy of this feature. + # ⚠ DEGRADE, NEVER REFUSE, which is the posture `columns()`'s own docstring sets: without the + # commission table the population falls back to the FLAGGED partners alone and `commissioned` + # reads blank for every row — fewer agents and an honestly empty column, rather than no spawn. + has_comm = bool(columns(cur, "account_invoice_line_agent")) + commissioned = ("(p.id IN (SELECT agent_id FROM account_invoice_line_agent " + " WHERE agent_id IS NOT NULL))" if has_comm else "FALSE") + union_leg = (" SELECT agent_id FROM account_invoice_line_agent WHERE agent_id IS NOT NULL " + " UNION " if has_comm else "") + sql = (f"SELECT p.id, p.name, {flagged}, {commissioned} AS commissioned " + "FROM res_partner p WHERE p.id IN (" + f"{union_leg}SELECT id FROM res_partner WHERE {flagged})") + out = [] + for (aid, name, flag, comm) in cur.execute(sql).fetchall(): + out.append({ + "_id": str(aid), + "agent": str(name or ""), + "odoo_id": int(aid), + AGENT_JOIN_KEY: int(aid), + "flagged": "1" if flag else "", + "commissioned": "1" if comm else "", + }) + return out + + +def read_accounts(cur): + """[(row dict)] — the whole GL chart, keyed on the `account.account` id. + + ⚠ THE EXPENSE PREDICATE IS THE SEMANTIC LAYER'S, copied rather than invented: + `harness/semantic.py`'s `gl_lines` topic scopes expenses as + `account_type in ('expense','expense_depreciation')`. A second definition here is how a + column and a topic start disagreeing about the same word. + ⛔ `account.account` has NO `active` column in this Odoo version (a domain naming it 500s), so + there is nothing to filter and every account is a row. + """ + have = columns(cur, "account_account") + if not have: + return [] + sql = (f"SELECT id, {_col(have, 'code', chr(39) + chr(39))}, " + f" {_col(have, 'name', chr(39) + chr(39))}, " + f" {_col(have, 'account_type', chr(39) + chr(39))} FROM account_account") + out = [] + for (aid, code, name, atype) in cur.execute(sql).fetchall(): + t = str(atype or "") + out.append({ + "_id": str(aid), + ACCOUNT_JOIN_KEY: str(code or ""), + "account_name": str(name or ""), + "odoo_id": int(aid), + "account_type": t, + "is_expense": "1" if t in ("expense", "expense_depreciation") else "", + }) + return out + + +_VENDOR_DOCS = "state = 'posted' AND move_type IN ('in_invoice','in_refund')" + + +def read_bills(cur): + """[(row dict)] — posted vendor bills and refunds, keyed on the `account.move` id.""" + have = columns(cur, "account_move") + if not have: + return [] + sql = ("SELECT id, name, partner_id, partner_name, invoice_date, invoice_date_due, " + f" {_col(have, 'amount_untaxed_signed', '0')}, " + f" {_col(have, 'amount_residual_signed', '0')}, " + f" {_col(have, 'payment_state', chr(39) + chr(39))}, move_type " + f"FROM account_move WHERE {_VENDOR_DOCS} AND partner_id IS NOT NULL") + out = [] + for r in cur.execute(sql).fetchall(): + (mid, name, pid, pname, when, due, untaxed, residual, pay, mtype) = r + out.append({ + "_id": str(mid), + "bill_no": str(name or ""), + "odoo_id": int(mid), + "vendor": str(pname or ""), + VENDOR_JOIN_KEY: int(pid), + "invoice_date": _as_date(when), + "due_date": _as_date(due), + "amount_untaxed": float(untaxed or 0.0), + "residual": float(residual or 0.0), + "payment_state": str(pay or ""), + "move_type": str(mtype or ""), + }) + return out + + +def read_vendors(cur): + """[(row dict)] — every partner carrying a posted vendor bill, keyed on the `res.partner` id. + + ⚠ DERIVED FROM THE BILLS, unlike `read_customers` which is deliberately NOT derived from its + invoices. The asymmetry is intentional and the reason is what that function's own comment + says: a customer registry sourced from receivables is what kept most Odoo ids out of the store. + There is no second document universe for vendors — a partner with no bill has no payable + history to show — so the bill IS the population, and MEASURED it dangles nothing (0 bills + carry a null partner; all 393 vendors resolve in `res_partner`). + """ + have = columns(cur, "res_partner") + if not have or not columns(cur, "account_move"): + return [] + # ⭐⭐ W33-T48 (owner item 13). The grid served FIVE columns off a partner the census measured + # at SEVENTY-SIX populated fields. The nine added here are the ones a buyer actually asks a + # vendor record for — how to reach them, who they are for tax, and where they are. + # ⛔ EVERY ONE GOES THROUGH `_col`, which substitutes a literal when the mirror lacks the + # column. That is not defensive habit: this projection has to keep working against a mirror + # that has not been re-synced since the widening, and the alternative is a reader that raises + # on the exact box the widening was meant to help. The columns arrive when the backfill runs; + # until then these read blank rather than failing. + blank = chr(39) + chr(39) + sql = ("SELECT p.id, p.name, " + f"{_col(have, 'p.country_name', blank)}, {_col(have, 'p.email', blank)}, " + f"{_col(have, 'p.phone', blank)}, {_col(have, 'p.mobile', blank)}, " + f"{_col(have, 'p.website', blank)}, {_col(have, 'p.vat', blank)}, " + f"{_col(have, 'p.ref', blank)}, {_col(have, 'p.street', blank)}, " + f"{_col(have, 'p.street2', blank)}, {_col(have, 'p.city', blank)}, " + f"{_col(have, 'p.zip', blank)} " + "FROM res_partner p WHERE p.id IN " + f" (SELECT partner_id FROM account_move WHERE {_VENDOR_DOCS} " + " AND partner_id IS NOT NULL)") + out = [] + for (pid, name, country, email, phone, mobile, website, vat, ref, + street, street2, city, zipc) in cur.execute(sql).fetchall(): + out.append({ + "_id": str(pid), + "vendor": str(name or ""), + "odoo_id": int(pid), + VENDOR_JOIN_KEY: int(pid), + "country": str(country or ""), + "email": str(email or ""), + "phone": str(phone or ""), + "mobile": str(mobile or ""), + "website": str(website or ""), + "vat": str(vat or ""), + "vendor_ref": str(ref or ""), + "street": str(street or ""), + "street2": str(street2 or ""), + "city": str(city or ""), + "zip": str(zipc or ""), + }) + return out + + +def customers_from(invoice_rows): + """The partners carrying the given invoice rows — the pre-2026-08-09 population builder. + + ⚠ NO LONGER WHAT SPAWNS `ut_odoo_customers` (that is `read_customers`). Kept because it is a + pure function over rows and the gate uses it to prove the FOLD against a fixture without a + mirror; deleting it would cost a test its independence from the SQL. + """ + out = {} + for row in invoice_rows: + pid = row[JOIN_KEY] + entry = out.setdefault(str(pid), {"_id": str(pid), "customer": row["customer"], + JOIN_KEY: pid}) + # A partner's name can differ across documents (renames land on new invoices only); + # the newest non-empty one wins so the locked table shows what Odoo shows today. + if row["customer"]: + entry["customer"] = row["customer"] + return list(out.values()) + + +# --------------------------------------------------------------------------------------------- +# THE SPAWN +# --------------------------------------------------------------------------------------------- +class Refused(Exception): + """A refusal a caller should SHOW, not swallow. Every raise names what would otherwise have + been written wrong.""" + + +#: `plan()` bucket -> (store key, nav label, field contract). ⭐ ONE ROW PER TABLE is the whole +#: point: adding an Odoo entity is a spec row plus a reader, not a fifth copy of the spawn code. +#: ⚠ ORDER MATTERS ONLY FOR THE REFUSAL MESSAGE; `plan` checks every cap before anything commits. +TABLES = ( + # ⛔⛔ `customers` AND `products` ARE GONE FROM THIS TUPLE — W33-T44 / owner item 12 / + # AMENDMENT A2. They presented the same SUBJECTS as the compiled registry modules + # `customer_data` and `product_data` ("why do we have 'Odoo products' already with the current + # Products database? The Unique ID is redundant"), and R2 ruled the LEGACY key survives: it + # keeps its store bucket, so no saved view, grant, cohort or formula moves, and it now carries + # the twins' join keys (`partner_id`; `product_id` pending the ask in `mailbox/E.md`). + # Dropping the row here is what stops them being planned, built or re-created; + # `RETIRED_KEYS` below is what removes the rows a tenant already has. + ("invoices", INVOICES_KEY, "Odoo invoices", invoice_fields), + ("orders", ORDERS_KEY, "Odoo orders", order_fields), + # ⭐ WAVE 28 / R1. Measured populations: 19 / 192 / 6,538 / 393 — every one of them two orders + # of magnitude inside `MAX_ROWS`, which is why the answer to "every unique id is a database" + # is four more spec rows and four readers rather than a new substrate. + ("agents", AGENTS_KEY, "Odoo agents", agent_fields), + ("accounts", ACCOUNTS_KEY, "Odoo GL accounts", account_fields), + ("vendors", VENDORS_KEY, "Odoo vendors", vendor_fields), + ("bills", BILLS_KEY, "Odoo vendor bills", bill_fields), + # ⭐⭐ W30-T35 / R7 — the two READ-THROUGH grains. They are spec rows like any other, and that + # is the point: `apply_plan` creates their DEFINITION (label, fields, lock, nav entry, grants) + # exactly as it does for the eight above, and `plan` hands them ZERO rows. Leaving them out of + # this tuple was the alternative and it is the wrong one — the route 404s on a key `TABLES` + # does not name, so the grids would be bound to the mirror and unreachable, which is this + # wave's own [[reachable-is-not-the-same-as-built]] shape. + ("order_lines", ORDER_LINES_KEY, "Odoo order lines", order_line_fields), + ("gl_lines", GL_LINES_KEY, "Odoo GL lines", gl_line_fields), +) + +#: store key -> the REAL-WORLD POPULATION that table presents (`core.registry`'s `subject` +#: vocabulary, same strings, one namespace). ⭐ W33-T41 / item 12a. +#: +#: ⛔ A SEPARATE MAP RATHER THAN A FIFTH ELEMENT ON EACH `TABLES` ROW, and that is not tidiness: +#: six sites in this file and two in the gate unpack `for bucket, key, _l, _f in TABLES`, so +#: widening the tuple is eight edits that all fail loudly at once and one — in a gate — that +#: would fail QUIETLY, having already been rewritten to match. The subject is a fact ABOUT the +#: key, and this is the shape that says so. +#: +#: ⚠ THE THREE res.partner SUBSETS ARE THREE SUBJECTS, NOT ONE. Customers, agents and vendors all +#: read `res_partner`, and they are different POPULATIONS of it — the claim is over who is in the +#: database, never over which Odoo model was queried. Same for `account_move`, which is the +#: customer-invoice book under one WHERE and the vendor-bill book under another. +TABLE_SUBJECTS = { + # ⛔ `CUSTOMERS_KEY` and `PRODUCTS_KEY` are absent — W33-T44 retired them, and their subjects + # (`odoo:res.partner`, `odoo:product.product`) are claimed by `core.registry`'s `customer_data` + # and `product_data` rows, which is now the ONLY claim on each. That is item 12 satisfied: one + # subject, one database, and `subject_conflict` would REFUSE either of these keys if a future + # spec row tried to bring it back. + INVOICES_KEY: "odoo:account.move.customer", + ORDERS_KEY: "odoo:sale.order", + AGENTS_KEY: "odoo:res.partner.agent", + ACCOUNTS_KEY: "odoo:account.account", + VENDORS_KEY: "odoo:res.partner.vendor", + BILLS_KEY: "odoo:account.move.vendor", + ORDER_LINES_KEY: "odoo:sale.order.line", + GL_LINES_KEY: "odoo:account.move.line", +} + +#: ⛔⛔ THE TWO RETIRED KEYS — W33-T44 / AMENDMENT A2. The rows a tenant ALREADY HAS. +#: +#: Dropping the `TABLES` rows above stops these being planned or re-created; it does NOT remove the +#: definitions and rows already sitting in a tenant's `user_tables` document, which is what the +#: owner actually sees in the nav. This set is what removes them, and it is applied inside +#: `apply_plan`'s single atomic updater — i.e. BY THE CONTAINER, on the boot rebuild and every +#: resync. +#: +#: ⛔ IT MUST BE THE CONTAINER AND NOT A CLI, AND THIS IS MEASURED, NOT CAUTIOUS (D-195): a +#: developer's script CAN write the tenant store, the write returns clean, a fresh read confirms +#: it — and the running Space reverts it within a minute, because the container holds the document +#: and re-uploads its own copy (download-modify-upload, last write wins). A removal shipped as a +#: script is a dry run that reports success. +#: +#: ⛔ AND A SWEEP THAT DELIVERS MUST NOT CREATE. Wave 32's `ut_ensure` was handed a merge-only job +#: and minted 8 empty databases in every tenant, because the door it used creates when absent. This +#: is a `pop`, it runs only over keys already present, and the gate asserts BOTH halves — removed, +#: AND not re-created on the next pass. Gating only the removal would pass on a tree that deletes +#: and re-adds the table every 30 minutes. +RETIRED_KEYS = (CUSTOMERS_KEY, PRODUCTS_KEY) + +#: ⛔ THE GRANDFATHER LIST IS DELETED — W33-T44 did what it was written to force. +#: +#: It existed for exactly one wave, to keep the product runnable between W33-T41 (the uniqueness +#: check) and W33-T44 (the retirement): the check is CORRECT and the collision it forbids was +#: LIVE, so without a named exemption the spawn refused on any fresh document and tenant #0 wrote +#: nothing at all. The exemption was ratcheted BOTH ways — a collision outside it was a new +#: duplicate, a member that stopped colliding was a stale exemption — so retiring the twins turned +#: the gate RED until this constant went with them. It did, and that is the ratchet working. +#: `verify_odoo_relational::_prove_subject_uniqueness` now asserts the collision set is EMPTY. + +#: The bucket -> reader map. ⛔ ITS ABSENCES ARE LOAD-BEARING: a bucket with no reader has no +#: python row builder ANYWHERE, which is what makes "never materialised" structural rather than a +#: policy `plan()` could forget. The two line grains are absent for that reason and no other. +_READERS = { + "customers": lambda cur, excluded: read_customers(cur, excluded=excluded), + "products": lambda cur, excluded: read_products(cur), + "invoices": lambda cur, excluded: read_invoices(cur, excluded=excluded), + "orders": lambda cur, excluded: read_orders(cur, excluded=excluded), + "agents": lambda cur, excluded: read_agents(cur), + "accounts": lambda cur, excluded: read_accounts(cur), + "vendors": lambda cur, excluded: read_vendors(cur), + "bills": lambda cur, excluded: read_bills(cur), +} + +#: The table keys this module can never materialise — DERIVED from the absence of a reader, never +#: typed out, so it cannot drift from the fact it describes. +#: +#: ⛔⛔ IT IS STAMPED ONTO THE DEFINITION AT SPAWN, AND THAT IS NOT BELT-AND-BRACES — IT IS THE +#: ONLY WAY THESE TWO TABLES EVER GET THE DURABLE FLAG. `core.user_tables.materialises` reads a +#: process-global registry first and falls back to a stored `readThrough` stamp, "which is what a +#: cold process reads" — but the only writer of that stamp is `strip_materialised`, and it stamps +#: exclusively tables it found rows on (`if isinstance(t, dict) and t.get('rows')`, after an early +#: return when nothing is fat). A table that was BORN read-through has no rows to strip, so it is +#: never stamped, so a process that cannot reach the mirror reads `rows: {}` and calls that the +#: answer — an EMPTY GRID with nothing going red, which is the exact failure that docstring names. +#: The conversion writes the stamp; a table that needs no conversion still needs the statement. +READ_THROUGH_KEYS = frozenset(key for bucket, key, _l, _f in TABLES if bucket not in _READERS) + + +class _LentDoc: + """A store handle that serves the ONE `user_tables` document `plan()` has ALREADY read. + + ⛔⛔ THIS IS NOT A MICRO-OPTIMISATION AND IT IS NOT OPTIONAL. `core.user_tables.row_limit` + resolves through `materialises` → `get` → `all_tables(st)`, and every one of those is a WHOLE + 20 MB document read, deep-copied under `Store._lock`. `plan()` asks the evaluator once per + table per loop, so passing the live handle would have added ~16 full document copies to a + function that already reads it exactly once — and with `st=None` (the gate's fixture posture, + and any dry run) those reads resolve to the MODULE-GLOBAL store, i.e. a Hugging Face dataset + fetch per table, on a path that has no business touching the network at all. + `materialises`' own docstring asks callers to lend the definition they are holding; `row_limit` + takes `st` rather than `defn`, so the lending happens one level up, here. + + ⚠ It answers ONLY the user-tables document and `None` for anything else, deliberately: a shim + that quietly proxied other keys would be a second store with a partial view, which is worse + than one that says what it knows. + """ + + def __init__(self, doc, key): + self._doc, self._key = doc if isinstance(doc, dict) else {}, key + + def get(self, name): + return self._doc if name == self._key else None + + +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ W32-T15/T16/T17 / CONTRACT C2 / RULINGS R9, R10, R11 — THE CONNECTOR'S OWN CONFIGURATION +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# +# The owner opened the Odoo connector and found nothing to configure: no key, no server database, +# no choice of which grids to materialise, no sync cadence, no way off. R9/R10/R11 answer all +# four, and the state lives HERE rather than in the route because `refresh()` is what has to obey +# it — a config the route knows and the sync path does not is a switch that flips nothing. +# +#: `{"grids": {table_key: bool}, "syncEvery": "", "frozen": bool, "frozenAt": ""}` +CONFIG_KEY = "odoo_connector_config" + +#: ⭐ R11's presets, and the FLOOR IS THE POINT. Owner: *"30m / 1h / 4h / daily / manual. The +#: floor is 30 minutes, server enforced; no free-text interval."* An interval box would let a +#: tenant ask for 60 s against an ERP over XML-RPC and a Hugging Face free-tier container. +#: ⚠ `manual` is not "very slow" — it is NO scheduled sync at all, which is why it maps to None +#: rather than to a large number. A caller that treats None as a duration gets a TypeError rather +#: than a silent once-a-century schedule. +SYNC_PRESETS = {"30m": 1800, "1h": 3600, "4h": 14400, "daily": 86400, "manual": None} +SYNC_FLOOR_SECONDS = 1800 +DEFAULT_SYNC = "30m" + + +def read_config(rt): + """This tenant's stored connector config, defaulted. Never raises — a store hiccup must not + make a connector look disconnected.""" + try: + cur = rt.get(CONFIG_KEY) if rt is not None else None + except Exception: # noqa: BLE001 + cur = None + cur = cur if isinstance(cur, dict) else {} + grids = cur.get("grids") if isinstance(cur.get("grids"), dict) else {} + every = cur.get("syncEvery") + return {"grids": {str(k): bool(v) for k, v in grids.items()}, + "syncEvery": every if every in SYNC_PRESETS else DEFAULT_SYNC, + "frozen": bool(cur.get("frozen")), + "frozenAt": str(cur.get("frozenAt") or "")} + + +def grid_choices(rt): + """`[{key, label, enabled}]` for every grid this connector can materialise — DERIVED from + `TABLES`, never a second hand-typed list (contract C2's parity leg asserts exactly that). + + ⚠ ABSENT MEANS ENABLED. A tenant that has never opened the panel has every grid, which is + what they have today; only an explicit untick turns one off. The alternative — an empty + config meaning "nothing enabled" — would silently unspawn ten live databases on deploy. + """ + chosen = read_config(rt)["grids"] + return [{"key": key, "label": label, "enabled": bool(chosen.get(key, True))} + for _bucket, key, label, _fields in TABLES] + + +def enabled_buckets(rt): + """The BUCKET names `plan()` speaks, for the grids this tenant has left ticked.""" + chosen = read_config(rt)["grids"] + return {bucket for bucket, key, _l, _f in TABLES if chosen.get(key, True)} + + +def sync_seconds(rt): + """How often this tenant's Odoo mirror should resync, or None for `manual` (R11). + + ⛔ THE FLOOR IS ENFORCED HERE AS WELL AS AT THE WRITE DOOR, deliberately. A stored value that + predates the preset list, or one written by any path that is not the route, must still not be + able to ask this loop for a 60-second cycle — a limit with only one enforcer is a limit that + holds until somebody finds the second way in [[limit-with-no-enforcer]]. + """ + secs = SYNC_PRESETS.get(read_config(rt)["syncEvery"], SYNC_PRESETS[DEFAULT_SYNC]) + if secs is None: + return None + return max(int(secs), SYNC_FLOOR_SECONDS) + + +def frozen(rt): + """Has this tenant DISCONNECTED Odoo (R10)? Frozen grids keep every row and every field and + stop being refreshed — distinct from PAUSED, which is temporary and keeps the credential.""" + return bool(read_config(rt)["frozen"]) + + +def plan(cur, rt=None): + """The rows that WOULD be written, plus the refusals that apply — no store WRITE at all. + + Separated from `apply_plan` so a route, a gate and a dry run all measure the same thing, and + so **every cap is checked before anything is committed**. + + Returns one key per bucket plus two that are not buckets: `problems` (refusals — a non-empty + list makes `apply_plan` raise before it writes anything) and, since W30-T35, **`limits`** — + `{table_key: limit_report}` for every table this plan did not fully materialise, which is R6's + second sentence carried as data rather than left for a reader to infer from an empty list. + + ⛔ `rt` IS WHAT MAKES THE TABLE-COUNT CHECK HONEST, and leaving it out was a real half-spawn + bug. This spawn writes FOUR tables in four updater passes; a tenant near `MAX_TABLES` would + create some and refuse the rest — leaving a locked invoices database with no rollup host, + while the route answered as though nothing had happened. A partial spawn is worse than a + refused one, so the count is checked against the tables that ALREADY exist, before the first + write. `rt` also carries the stored row counts the shrink guard compares against. + """ + ut = _ut() + # ⭐ R6, AND WITHOUT THIS LINE THE RULE IS ONLY ACCIDENTALLY TRUE. `is_connected` answers from + # three places in falling authority: the registry, a stored `connected: True`, then the + # `ut_odoo_` naming convention — and that last leg needs the table to ALREADY EXIST. So on a + # FIRST spawn, in a process that has not yet built `routes_odoo_tables.GRID_SOURCES`, every one + # of these tables reads as unconnected and earns `MAX_ROWS`: R6's cap removal would silently + # not apply on exactly the run that creates the databases. This module DECLARES these keys, so + # it is the honest place to say what they are. Idempotent (a set add), and it fills the + # evaluator's input rather than becoming a second evaluator. + ut.register_connected(*[key for _b, key, _l, _f in TABLES]) + excluded = excluded_ids(cur) + # ⭐⭐ W30-T35 / R6 / R7 — WHICH BUCKETS ARE BUILT AT ALL IS NOW ASKED, NOT ASSUMED, and it is + # `core.user_tables.row_limit` that answers: 0 = "stores no rows HERE" (read-through), None = + # "connected and uncapped", MAX_ROWS = "the editable substrate". Its own docstring names this + # function as the caller that reads it, which is the seam working as designed — one evaluator, + # so the spawn, the write doors and the wire cannot disagree about whether a table is capped + # ([[one-evaluator-per-question]]). + # + # ⛔ TWO DIFFERENT REASONS NOT TO BUILD, AND THEY ARE KEPT SEPARATE ON PURPOSE: + # * no reader at all — structural, permanent, and the case that must not depend on a store + # read succeeding (a cold process with no mirror still must not try to build 963,783 rows); + # * a reader exists but the table has already been converted to read-through — the + # `ut_odoo_accounts` case. Building 192 rows and letting `strip_materialised` delete them + # again on the next pass "works", and it is exactly the wasted, dangerous work `row_limit` + # was built to prevent. It also stops a refresh from silently RE-MATERIALISING a table + # D-87's conversion had already emptied. + # + # ⚠ THE DOCUMENT IS READ **ONCE**, HERE, AND LENT TO THE EVALUATOR — see `_LentDoc`. It used + # to be read after the build loop; it moved up because `row_limit` needs it and reading it per + # table per loop is ~16 more whole-document deep copies (or, with `st=None`, a Hugging Face + # fetch per table on a path that must never touch the network). + existing = {} + if rt is not None: + try: + existing = dict(rt.get(ut.STORE_KEY) or {}) + except Exception: # noqa: BLE001 + existing = {} + # ⛔ THE LENT DOCUMENT CARRIES THE `readThrough` STAMP THIS MODULE IS RESPONSIBLE FOR, and + # without it R6's report is silently absent on the run that matters most — the FIRST spawn. + # MEASURED: with an empty store, `row_limit` finds no registry entry and no stored stamp, so + # it answers `None` ("connected and uncapped") for a grain that stores nothing at all, and + # `limit_report` answers None with it — so `plan()["limits"]` came back EMPTY and the grids + # were skipped with no stated reason. The structural `reader is None` guard still did its job; + # what went missing was the half of R6 that has to SAY WHY. + # + # ⚠ THE OBVIOUS FIX IS THE ONE I DID NOT TAKE: `ut.register_read_through(*READ_THROUGH_KEYS)` + # would work in one line, and `core.user_tables` explicitly reserves that registrar — + # *"IT IS ALSO THE ONLY PLACE THAT MAY CALL `register_read_through`"* — for + # `routes_odoo_tables.sync_read_through`, because ELIGIBILITY needs the mirror and the fold + # matrix. That reasoning does not apply to a grain with no reader (there is nothing it could + # be eligible FOR), but the law is written without an exception, so this lends the evaluator + # the definition instead of taking one. `_ensure_table_inplace` writes exactly this stamp, so + # what is lent is the document as it stands the moment this plan applies. + lent = _LentDoc({**existing, + **{k: {**(existing.get(k) or {}), "readThrough": True} + for k in READ_THROUGH_KEYS}}, + ut.STORE_KEY) + + built, limits = {}, {} + caps = {} + for bucket, key, _label, _fields in TABLES: + reader = _READERS.get(bucket) + caps[key] = cap = ut.row_limit(key, st=lent) + if reader is None or cap == 0: + built[bucket] = [] + report = ut.limit_report(key, st=lent) + if report: + limits[key] = report + continue + built[bucket] = reader(cur, excluded) + problems = [] + + if rt is not None: + needed = [k for _b, k, _l, _f in TABLES if k not in existing] + if needed and len(existing) + len(needed) > ut.MAX_TABLES: + problems.append( + f"tenant holds {len(existing)} of MAX_TABLES={ut.MAX_TABLES} user tables and " + f"needs {len(needed)} more ({', '.join(needed)}); refusing rather than creating " + f"part of a linked set") + + for bucket, key, _label, _fields in TABLES: + rows = built[bucket] + cap = caps[key] # asked ONCE per table, above — never re-read per loop + # ⭐⭐ R6: THE `MAX_ROWS` REFUSAL IS GONE FOR A CONNECTED SOURCE, AND THE SENTENCE IT USED + # TO PRINT IS NOW `limit_report`'s STRUCTURED ANSWER. Owner, verbatim: *"there is no cap in + # how many data from the API source (as long as its from a connected source like Odoo) that + # can be pulled into the app… Now if there is lag or it can't be done, you need to + # explicitly tell me why and recommend a fix."* Both halves are here: a connected table + # answers `None` and is never refused for its size, and anything that IS still bounded is + # reported with its cause and its recommendation instead of a hand-typed line. + # + # ⛔ THE `cap and` GUARD IS THE WHOLE CHANGE AND ITS TWO FALSY CASES MEAN OPPOSITE THINGS: + # `None` = connected, uncapped, build every row Odoo has; `0` = stores no rows here, and + # the loop above already handed it an empty list. Neither may reach the refusal. The + # editable substrate still gets `MAX_ROWS` and is still REFUSED, never truncated — a capped + # table understates every total it feeds while looking exactly like a complete one. + if cap and len(rows) > cap: + report = ut.limit_report(key, st=lent) or {} + limits[key] = report + problems.append( + f"{key}: {len(rows):,} rows exceeds the {cap:,}-row ceiling; refusing " + f"({report.get('cause', 'a truncated table understates every rollup it feeds')}). " + f"{report.get('recommendation', '')}".strip()) + # ⚠ THE SHRINK GUARD SKIPS A READ-THROUGH GRAIN, and without this it would refuse every + # spawn after the first conversion: zero rows against a stored population is the INTENDED + # end state there, not the partial mirror read this guard exists to catch. + if cap == 0: + continue + stored = len(((existing.get(key) or {}).get("rows")) or {}) + if stored and len(rows) < stored * MAX_SHRINK: + problems.append( + f"{key}: the mirror answered {len(rows)} rows against {stored} stored — a drop of " + f"more than {int((1 - MAX_SHRINK) * 100)}% is a bad read, not Odoo history " + f"shrinking; refusing rather than deleting rows that still exist") + built["problems"] = problems + # R6's second sentence as DATA rather than prose: every table whose rows this plan did not + # (or may not) materialise, with the cause and the recommendation `core.user_tables` derives. + # ⚠ NOT a bucket — `apply_plan` iterates `TABLES` and asks `if bucket in built`, so a key that + # is not a bucket name is inert there, exactly as `problems` has always been. + built["limits"] = limits + return built + + +def apply_plan(rt, built, username="automation", today=None, report=None): + """Create-or-merge every table in `built` and its rows. Idempotent by construction. + + Row ids ARE the Odoo ids, so a re-run updates in place and never appends a second copy of the + same record — which is also what makes "every Odoo unique id is in the database" a checkable + statement rather than a hopeful one. + + ⚠ ONLY THE BUCKETS PRESENT ARE WRITTEN, so a caller (or a gate) may hand in a subset. + """ + if built.get("problems"): + raise Refused("; ".join(built["problems"])) + stamp = today or _iso_today() + written = {} + # ⚠ An OPTIONAL out-parameter, not a return-shape change: `written` has one value shape and + # keeps it. A caller that wants to SAY what was retired passes a dict; `refresh` does. + retired = [] + plans = [(key, label, fields(), built[bucket]) + for bucket, key, label, fields in TABLES if bucket in built] + + # ⭐⭐ ONE SYNC WRITE FOR ALL FOUR TABLES, not one per table — measured, not tidied. + # + # ⛔ A `flush="sync"` update of `user_tables` is a FULL DOWNLOAD of the document plus a full + # UPLOAD of it (`Store.update` -> `_read_strict` -> `put`). Four of them against the 20.6 MB + # document these tables produce is ~165 MB of Hugging Face traffic and four dataset commits + # EVERY resync — and `main.py` runs this at boot and after every `sync_all()` (~30 min). + # Composed into one pass it is ~41 MB and one commit: the same rows, a quarter of the bill. + # + # ⭐ AND IT IS ATOMIC, WHICH IS THE BIGGER WIN. `_ensure_table_inplace` raises `Refused` at + # `MAX_TABLES`; with four separate writes that refusal landed AFTER earlier tables had + # already been committed, leaving exactly the half-spawn `plan()` opens by refusing to + # create. Inside one updater, a raise aborts before anything is persisted. + def _apply_all(cur): + cur = cur if isinstance(cur, dict) else {} + # ⛔⛔ W33-T44 — THE RETIREMENT, INSIDE THE SAME ATOMIC WRITE. This updater is the ONE + # sync write the whole spawn makes, and it runs in the CONTAINER (boot rebuild + every + # resync), which is the only place a change to the tenant document survives (D-195: the + # same edit from a CLI reports success and is reverted within a minute). + # ⚠ It POPS and never creates: `RETIRED_KEYS` are absent from `TABLES`, so `plans` cannot + # contain them, and there is no door here that could re-add one. `removed` is reported so + # a caller can SAY what happened rather than infer it from a table going missing. + # ⛔ THE REMOVED KEYS DO **NOT** GO INTO `written`, and the first cut of this put them + # there. `written` maps table key -> a COUNTS dict, and every consumer iterates its + # `.values()` expecting `c["added"]`; a list under `"_retired"` made the very next leg die + # with `TypeError: list indices must be integers`. One dict, two value shapes, is a + # sentinel in a result set [[sentinel-in-a-sort-key]] — so the retirement reports through + # its OWN channel and the return type stays exactly what it was. + for key in RETIRED_KEYS: + if cur.pop(key, None) is not None: + retired.append(key) + for key, label, fields, rows in plans: + written[key] = _ensure_table_inplace(cur, key, label, fields, rows, username, stamp) + return cur + + rt.update(_ut().STORE_KEY, _apply_all, flush="sync") + if report is not None and retired: + report["retired"] = sorted(retired) + return written + + +def _ensure_table(rt, key, label, fields, rows, username, stamp): + """One table, written on its own. Kept because the gate drives a single table directly, and + because a caller with one table to reconcile should not have to compose an updater.""" + written = {} + + def _one(cur): + cur = cur if isinstance(cur, dict) else {} + written["counts"] = _ensure_table_inplace(cur, key, label, fields, rows, username, stamp) + return cur + + rt.update(_ut().STORE_KEY, _one, flush="sync") + return written["counts"] + + +def _ensure_table_inplace(cur, key, label, fields, rows, username, stamp): + """One table INSIDE a caller's updater: definition merged, rows reconciled, dict mutated. + + ⚠ ROWS THAT LEFT THE POPULATION ARE REMOVED, and that stayed correct through the widening — + but only because the populations widened to "everything Odoo has". While `ut_odoo_customers` + was built FROM open invoices, removal meant a customer who paid their bill vanished from the + registry; now a partner leaves only when their last document does. The shrink guard in + `plan()` is the backstop for the case this policy cannot distinguish: a partial mirror read. + """ + ut = _ut() + wanted = {r["_id"]: {k: v for k, v in r.items() if k != "_id"} for r in rows} + for row in wanted.values(): + row["refreshed"] = stamp + counts = {"added": 0, "updated": 0, "removed": 0, "rows": len(wanted)} + subject = TABLE_SUBJECTS.get(key) + table = cur.get(key) + if table is None: + if len(cur) >= ut.MAX_TABLES: + # ⚠ `ut_ensure` returns silently at this cap; a silent no-op here would report a + # successful refresh over a table that does not exist. + raise Refused(f"{key}: tenant is at MAX_TABLES={ut.MAX_TABLES}; nothing created") + # ⭐⭐ W33-T41 / item 12a — THE UNIQUENESS CHECK, ON THE LINE THAT MINTED THE DUPLICATE. + # + # ⛔ IT GUARDS **CREATION ONLY**, AND THAT IS THE WHOLE DESIGN, NOT A WEAKENING. This + # function is the boot rebuild and the 1800 s resync; it adopts an existing table by key + # on every pass. A claim test outside this `if` refuses the table it created last boot, + # `_apply_all` raises inside the updater, and tenant #0 spawns NOTHING — the check would + # take the product down to prevent a duplicate that already exists. Refusing the SECOND + # birth is what "make sure this never happens" asks for; the FIRST one is T44's job to + # remove, and it is removed by dropping its `TABLES` row, not by a guard here. + # + # ⚠ Claims are gathered from THIS TENANT'S document (`cur`) plus the compiled registry. + # Never a module-level cache: one Space process serves every tenant, and two tenants both + # holding an "Odoo customers" is correct (that is D-169's shape, and it is not repeated + # here). A stored definition with no `subject` claims nothing. + held = {t["subject"]: k for k, t in cur.items() + if isinstance(t, dict) and t.get("subject")} + # ⭐ NO EXEMPTION ANY MORE. The grandfather list is deleted with the two tables it covered + # (W33-T44), so this is now the plain rule the owner asked for: one subject, one database. + other = _registry().subject_conflict(subject, key, claimed=held) + if other: + raise Refused( + f"{key}: refusing to create a second database for {subject!r} — {other!r} " + f"already presents it. One subject, one database (item 12a); if this table is " + f"meant to replace {other!r}, retire {other!r} first rather than shipping both") + table = cur[key] = { + "key": key, "label": label, "source": ut.AUTOMATION_SOURCE, + "createdBy": username, "created": stamp, "fields": [], "rows": {}, + # recordMode = a LOCKED database (item-3 nomenclature): no human may add or + # delete records, while fields stay addable. Odoo owns this population. + "recordMode": ut.AUTOMATION_RECORD_MODE, + } + table.setdefault("recordMode", ut.AUTOMATION_RECORD_MODE) + # W30-T35 — the durable "my rows are not in this document" statement, on the tables no + # conversion will ever stamp (see `READ_THROUGH_KEYS`). Written on every pass, not + # `setdefault`: it is derived from the code's own structure, so the code is what it must agree + # with, and a definition that somehow lost the flag should regain it rather than keep serving + # an empty grid. + if key in READ_THROUGH_KEYS: + table["readThrough"] = True + # W33-T41 — the claim, made DURABLE on the definition. Written on every pass for the same + # reason `readThrough` is: it is derived from this module's own structure, so the code is what + # it has to agree with, and a definition that lost the stamp should regain it rather than go + # on being invisible to the next table's claim test. ⚠ It is also the ONLY way the check sees + # a `ut_*` table at all — those are per-tenant DATA, never compiled registry rows, so a cold + # process reading a fresh document has nothing else to read the claim off. + if subject: + table["subject"] = subject + have = {str(f.get("key")): f for f in (table.get("fields") or [])} + for field in fields: + # ⛔⛔ THE ONE DOOR THAT BYPASSES THE FIELD VALIDATOR, HARDENED WHERE IT BYPASSES IT. + # + # `user_tables._clean_field` stamps `source: 'overlay'` unconditionally on create AND + # patch, so every field that goes through the normal door has one. This function does NOT + # go through that door — it writes definitions straight into the document — and + # `aios_grid.rows_from_pool` reads `field["source"]` as a HARD KEY, so a contract that + # ever omitted it would 500 the entire rows route rather than degrade one column. Measured + # today across 9 automation-owned tables and 173 fields: zero are missing it, so this is + # LATENT, not live (A's PENDING row, raised as D-9 and corrected by D-23 — the crash that + # prompted it came from a hand-written test fixture, not from production). + # + # ⚠ Fixed HERE rather than by softening `rows_from_pool` to `.get`, deliberately: a + # missing `source` means the definition does not say which stratum owns the column, and + # rendering it as blank would bury that. The default matches what the validator would + # have stamped, so the bypass stops being a hole without inventing a second rule. + if isinstance(field, dict) and not field.get("source"): + field = {**field, "source": "overlay"} + fkey = str(field.get("key")) + if fkey not in have: + table.setdefault("fields", []).append(dict(field)) + continue + # ⭐ A PRESET FIELD'S CONTRACT IS FORWARD-MIGRATED, not merely created once. The + # widening moved `payment_state`'s option list and every rollup's conditions; a + # create-only merge would have left the LIVE table declaring the old contract + # forever, so the column would render but its filter could not match what is stored. + # ⚠ Only machine-owned keys are touched — `automation.preset` is the wall — so a + # column a user added to a locked database is never rewritten. + stored = have[fkey] + # ⭐⭐ 2026-08-09 — a column a human has taken over keeps its own definition. Same stamp, + # same reader (`user_tables.user_edited`) and the same reason as the IG reconciler: the + # loop below overwrites `rollup` from the shipped contract, so an edited preset rollup on + # an Odoo database would silently revert at the next boot rebuild. + if _ut().user_edited(stored): + continue + if (stored.get("automation") or {}).get("preset"): + for prop in ("label", "type", "options", "link", "rollup", "description", + "agg", "pinned", "default"): + if prop in field: + stored[prop] = field[prop] + else: + stored.pop(prop, None) + stored_rows = table.setdefault("rows", {}) + for rid, values in wanted.items(): + current = stored_rows.get(rid) + if current is None: + stored_rows[rid] = dict(values) + counts["added"] += 1 + elif any(str(current.get(k, "")) != str(v) for k, v in values.items() + if k != "refreshed"): + current.update(values) + counts["updated"] += 1 + else: + current["refreshed"] = values["refreshed"] + for rid in [r for r in stored_rows if r not in wanted]: + stored_rows.pop(rid, None) + counts["removed"] += 1 + return counts + + +def is_royal(tenant): + return str(tenant or "").strip().lower() in RI_SLUGS + + +def refresh(rt, tenant, username="automation", cur=None, today=None): + """THE entry point — the store-resync path and the route both call this. + + ⚠ It must be CALLED on resync by something outside this file. If it is not wired, every row + still carries a `refreshed` stamp, so a stale worklist is at least LEGIBLE rather than + silently authoritative. + """ + if not is_royal(tenant): + raise Refused(f"tenant {tenant!r} has no Odoo mirror behind these tables (R1: Royal " + f"Imports only); refusing to spawn empty locked databases") + # ⭐⭐ W31-T45 / D-169 — THE SLUG GATE ABOVE AND THE FILE GATE HERE ANSWER DIFFERENT QUESTIONS, + # and this is the one place in the codebase where that is easy to miss. `is_royal` asks "is + # this tenant ENTITLED to Odoo databases"; it says nothing about WHICH DuckDB file this process + # has open. A worker pinned to another tenant's store (AIOS_DUCKDB_PATH, or a `use_path` in a + # provisioning script) passes `is_royal("royal-imports")` and then WRITES tenant #0's locked + # databases from another customer's rows — a spawn, not a read, so the wrong numbers become + # durable. Entitlement is not residency. + # ⚠ It runs when `cur` is LENT too, not only when we open one: the resync loop and the boot + # rebuild both hand a cursor in, and a lent cursor is exactly the case where nobody re-checks. + if rt is not None: + rt.assert_datastore_matches() + # ⛔⛔ W32-T16 / R10 — A DISCONNECTED CONNECTOR DOES NOT REFRESH, AND THAT IS THE WHOLE FREEZE. + # R10: *"Disconnect removes the credential and FREEZES the grids as static data."* Removing + # the credential alone is not a freeze — this function is also reached by the boot rebuild and + # the resync loop, and for tenant #0 the ENVIRONMENT still holds Odoo credentials, so a + # disconnected workspace would silently re-materialise from `.env` on the next tick and the + # "disconnect" would last until the container restarted. The refusal is a REPORT, not a raise: + # the resync loop calling this every cycle must not be handed an exception as a status. + if rt is not None and frozen(rt): + return {"tables": {}, "frozen": True, + "note": "this workspace has disconnected Odoo; its databases are frozen as " + "static data and are not being refreshed"} + if cur is None: + from harness import datastore + cur = datastore.ro_con() + built = plan(cur, rt=rt) + # ⭐ W32-T15 / R9 — THE GRID PICKER, ENFORCED WHERE IT COUNTS. `apply_plan` writes only the + # buckets present in `built`, so dropping an unticked one here is the whole of "unticking a + # grid stops it materialising on the next sync". Done AFTER `plan` rather than inside it so + # every cap, refusal and limit report is still computed over the full set — a config must not + # be able to hide a problem by hiding the table that has it. + # ⚠ It does NOT delete a grid that was already spawned. Unticking stops the next refresh from + # rewriting it; dropping the rows a tenant already has is `disconnect`'s job, and it does not + # do that either (R10 keeps them). Silent data deletion behind a checkbox is not on offer. + if rt is not None: + keep = enabled_buckets(rt) + skipped = sorted(key for bucket, key, _l, _f in TABLES if bucket not in keep) + for bucket, _key, _l, _f in TABLES: + if bucket not in keep: + built.pop(bucket, None) + else: + skipped = [] + # ⭐ W33-T44: a table this pass REMOVED is reported, not inferred from a grid going missing. + # Same rule as `skipped` below — R6's second sentence: what was deliberately not built (or no + # longer built) SAYS SO, with the keys. + plan_report = {} + written = apply_plan(rt, built, username=username, today=today, report=plan_report) + return {"tables": written, + **({"retired": plan_report["retired"]} if plan_report.get("retired") else {}), + # R6's second sentence: a set that was deliberately not built SAYS SO, with the keys. + **({"skipped": skipped} if skipped else {}), + **{bucket: len(built[bucket]) for bucket, _k, _l, _f in TABLES if bucket in built}} diff --git a/api/providers.py b/api/providers.py index 5425c862136396406714fecaa23eec0d3288ec52..d294400d33431e3b8c3bda9253357544cf451875 100644 --- a/api/providers.py +++ b/api/providers.py @@ -1,897 +1,897 @@ -"""THE PROVIDER LAYER — many vendors behind one capability, chosen per FIELD and per cost. - -⭐⭐ OWNER RULING 2026-08-08: *"Let's do a combination scraper. Do Bright Data first, and if -anything fails, we use APIfy or vice versa. Make this dynamic somehow. remember to cost optimize -when we scale to thousands+ customer … make sure our code is customizable and modular."* - -⛔ THE DESIGN DECISION THAT MATTERS, AND IT IS NOT "TRY A, THEN TRY B". -A provider-level failover ("if Bright Data errors, re-run the whole thing on Apify") is the obvious -shape and it is the expensive one. MEASURED: Bright Data answers profile, likes and comments -correctly and CANNOT answer view counts on any route; Apify answers view counts correctly. A -provider-level fallback would notice the missing views and re-buy the profile, the likes and the -comments from Apify as well — **paying twice for the 90% that already worked**. At one profile that -is noise; at thousands of customers × twelve posts each it is the whole bill. - -So routing is per CAPABILITY, and a capability is the smallest independently-billable thing: - - ig_profile -> brightdata (36 fields, works) - ig_post_metrics -> brightdata (likes + comments, works) - ig_post_views -> apify (Bright Data is INCAPABLE — see `capable=False` below) - ig_comments -> brightdata - -Each capability has an ORDERED list of providers. The first one that is configured AND capable AND -succeeds wins; the rest are never called, so the common path costs exactly one vendor record. - -⚠ "FAILURE" INCLUDES ANSWERING WITH THE FIELD BLANK. A vendor that returns HTTP 200 and a null in -the one column you asked for has failed for our purposes, and a chain that only catches exceptions -would stop at it forever. `run()` takes a `satisfied` predicate and falls through on an unsatisfying -answer exactly as it would on a 500. - -⚠ ORDER IS DATA, NOT CODE. `AIOS_PROVIDER_ORDER` overrides any chain at deploy time -(`ig_post_views=apify,brightdata;ig_profile=apify`), which is what "or vice versa" means and what -lets a tenant be moved off a vendor without a release. -""" -from __future__ import annotations - -import json -import os -import time -from dataclasses import dataclass, field -from typing import Callable - -# ⚠ TOP-LEVEL AND SAFE, unlike the `anthropic` import in `anthropic_sdk()`: `requests` is pinned in -# BOTH manifests and is already a transitive dependency of `huggingface_hub`, so it cannot be the -# line that stops a container booting. That asymmetry is the whole reason the two are imported -# differently — see `anthropic_sdk()`. -import requests - -#: ⚠ APPROXIMATE, AND DELIBERATELY SO. These are list prices per RECORD in USD, used only to RANK -#: providers and to estimate a run's spend for the operator. They are not billing truth — the -#: vendor's own dashboard is. Wrong by 2x still ranks correctly; wrong by 100x does not, which is -#: why they are named and dated rather than guessed silently. -#: Bright Data Instagram datasets ~ $0.0015/record (2026-08 list). Apify instagram-scraper -#: ~ $0.0027/result (2026-08 list). Both re-checked when a provider is added. -#: ⚠ WAVE 30 · T16 — APIFY CORRECTED 0.0023 -> 0.0027, and the direction matters: the old number -#: made the fallback look CHEAPER than it is, and `estimate()` is what a person is shown before -#: they authorise a run. An under-stated price is the one rounding error a cost guard cannot catch. -#: ⚠ Both are LIST rates used to rank and to estimate. They are not invoices, and nothing here -#: reads a live price — a modelled figure that says so is honest; one that pretends is not. -_COST_BRIGHTDATA = float(os.environ.get("AIOS_COST_BRIGHTDATA") or 0.0015) -_COST_APIFY = float(os.environ.get("AIOS_COST_APIFY") or 0.0027) - - -@dataclass(frozen=True) -class Capability: - """What ONE provider can do for ONE capability, and what it costs to ask.""" - #: ⛔ `False` means MEASURED INCAPABLE, not "untried". A provider declared incapable is never - #: called for this capability at all — it cannot be reached by a fallback, cannot be put first - #: by an env override, and cannot silently start being billed because somebody reordered a - #: list. Bright Data's `ig_post_views` is False on the strength of §4e-§4h: every route - #: (Posts x /p/, Posts x /reel/, Reels x /p/, Reels x /reel/, discover-by-profile) on four - #: accounts from 13K to 268M followers returned an account-grain constant or a null. - capable: bool = True - cost_per_record: float = 0.0 - #: Free-text, shown to an operator deciding where their money went. - note: str = "" - - -@dataclass -class Provider: - key: str - label: str - #: The env var holding this provider's credential. NEVER the credential itself — this module - #: is imported by surfaces that serialise their config. - key_env: str = "" - caps: dict = field(default_factory=dict) - - def configured(self) -> bool: - return bool((os.environ.get(self.key_env) or "").strip()) if self.key_env else True - - def cap(self, capability: str) -> Capability | None: - return self.caps.get(capability) - - def can(self, capability: str) -> bool: - c = self.cap(capability) - return bool(c and c.capable and self.configured()) - - -#: ⚠ ADDING A PROVIDER IS A REGISTRY ENTRY PLUS A RUNNER — no change to any caller. That is the -#: "modular/customizable" half of the ruling, and the reason the chains below name STRINGS. -PROVIDERS: dict[str, Provider] = { - "brightdata": Provider( - key="brightdata", label="Bright Data", key_env="AIOS_BRIGHTDATA_KEY", - caps={ - "ig_profile": Capability(True, _COST_BRIGHTDATA, "36 fields incl. bio/email/category"), - "ig_post_metrics": Capability(True, _COST_BRIGHTDATA, "likes + comments + captions"), - "ig_comments": Capability(True, _COST_BRIGHTDATA, "separate paid dataset, opt-in"), - # ⛔ THE ONE THAT COSTS US NOTHING TO GET RIGHT AND EVERYTHING TO GET WRONG. - "ig_post_views": Capability( - False, _COST_BRIGHTDATA, - "MEASURED INCAPABLE: returns one account-grain number for every reel of a " - "creator (identical for two shortcodes in one call, ticking upward between " - "calls) and never populates video_play_count. Their own documented example " - "account no longer reproduces it."), - # ⭐⭐ WAVE 29 (item 7 / D-9 / R1) — TIKTOK. Three datasets, all reachable with our - # existing key, all schema-probed for $0.00 (40/43/17 fields with the vendor's own - # types): `gd_l1villgoiiidt09ci` · `gd_lu702nij2f790tmv9h` · `gd_lkf2st302ap89utw5k`. - "tt_profile": Capability(True, _COST_BRIGHTDATA, - "40 fields incl. bio/engagement rates/region"), - # ⛔ THERE IS NO `tt_post_views` CAPABILITY, AND ITS ABSENCE IS THE CLAIM. - # `TikTok - Posts` declares `play_count: number` and the vendor's own sample says - # "no empty values or zeros" — so on TikTok the view count arrives INSIDE the post - # record and a separate view rung would be a second bill for a number we already have. - # ⚠ DECLARED, NOT MEASURED, and that distinction is the whole scar tissue of §4e: - # Bright Data's Instagram Reels also DECLARES `views: number` and delivers an - # account-grain wrong one. One live pull settles it (W29-T06, spend-gated). If it - # fails, the honest repair is a `tt_post_views` capability routed elsewhere — never a - # quiet fallback bolted onto this one. - "tt_post_metrics": Capability( - True, _COST_BRIGHTDATA, - "likes + comments + shares + saves, and play_count inline (DECLARED " - "no-empties-or-zeros, UNPROVEN until one live pull)"), - "tt_comments": Capability(True, _COST_BRIGHTDATA, - "separate paid dataset, opt-in: 17 fields"), - }), - "apify": Provider( - key="apify", label="Apify", key_env="AIOS_APIFY_KEY", - caps={ - # ⭐ MEASURED 2026-08-08 against the public Reels grid: `videoPlayCount` 137,684 and - # 299,493 vs a browser-read ground truth of 134K-137K and 299K. Exact. - "ig_post_views": Capability(True, _COST_APIFY, - "videoPlayCount, matches Instagram's displayed views"), - "ig_post_metrics": Capability(True, _COST_APIFY, "likes + comments (fallback)"), - "ig_profile": Capability(True, _COST_APIFY, "profile fields (fallback)"), - }), -} - -#: The DEFAULT chain per capability, cheapest-capable-first. Overridable — see `chain()`. -DEFAULT_CHAINS: dict[str, tuple] = { - "ig_profile": ("brightdata", "apify"), - "ig_post_metrics": ("brightdata", "apify"), - # Only one entry, and that is the point: Bright Data is declared incapable, so listing it here - # would be a lie that costs a wasted call on every single post. - "ig_post_views": ("apify",), - "ig_comments": ("brightdata",), - # ⭐ WAVE 29 — TikTok, and every chain is deliberately SINGLE-PROVIDER. - # ⛔ A MULTI-PROVIDER CHAIN IS A PROMISE SOMETHING WALKS IT. `ig_post_metrics` has declared a - # two-provider fallback since wave 28 and NOTHING reaches the second name: if Bright Data - # answers with the likes blank, Apify is never asked. `verify_automation`'s E2b pins that gap - # BY NAME so a third unwalked chain turns it red — which is exactly what a second name here - # would be today. Apify does sell TikTok (`clockworks/tiktok-profile-scraper`, $0.003/result, - # 0.7% 30-day failure rate — measured from its own store record), so the fallback is buildable; - # it is not declared until a runner walks it. - "tt_profile": ("brightdata",), - "tt_post_metrics": ("brightdata",), - "tt_comments": ("brightdata",), -} - - -def chain(capability: str) -> list: - """The provider order for `capability` — env override first, then the default. - - `AIOS_PROVIDER_ORDER` is a `;`-separated list of `capability=p1,p2` clauses. This is how the - owner's "or vice versa" is expressed WITHOUT a release, and how one tenant can be moved off a - vendor that is having a bad day. - ⛔ An override may REORDER and may DROP, but it can never make an incapable provider capable — - `can()` still gates every name. A chain that reads `ig_post_views=brightdata` therefore - resolves to EMPTY rather than to a provider that would return a wrong number, because a - plausible wrong number is worse than an honest refusal. - """ - raw = (os.environ.get("AIOS_PROVIDER_ORDER") or "").strip() - names = None - for clause in raw.split(";"): - if "=" in clause: - cap_name, _, order = clause.partition("=") - if cap_name.strip() == capability: - names = [x.strip() for x in order.split(",") if x.strip()] - if names is None: - names = list(DEFAULT_CHAINS.get(capability) or ()) - return [PROVIDERS[n] for n in names if n in PROVIDERS and PROVIDERS[n].can(capability)] - - -def estimate(capability: str, records: int) -> dict: - """What the FIRST capable provider would cost for `records` — the number an operator plans on. - - Reported per capability rather than per run because that is the unit that scales: at a thousand - tenants the question is never "what did this run cost", it is "what does adding view counts to - every post cost per month". - """ - ch = chain(capability) - if not ch: - return {"capability": capability, "records": records, "provider": None, "usd": 0.0, - "note": "no configured, capable provider"} - p = ch[0] - c = p.cap(capability) - return {"capability": capability, "records": records, "provider": p.key, - "usd": round((c.cost_per_record or 0.0) * max(0, int(records)), 4), - "note": c.note} - - -@dataclass -class Attempt: - provider: str - ok: bool - note: str = "" - records: int = 0 - seconds: float = 0.0 - - -def run(capability, work, satisfied=None, log=None): - """Walk the chain for `capability` until one provider gives a SATISFYING answer. - - `work(provider) -> (result, note)`; a non-empty note means it did not answer. - `satisfied(result) -> bool` decides whether an answer is good enough to stop. Default: any - truthy result. - - Returns `(result, attempts)`. `attempts` is the audit trail — every provider tried, whether it - satisfied, and how long it took — because "where did this number come from and what did it - cost" is a question somebody asks about a bill, not about a stack trace. - - ⛔ THE `satisfied` HOOK IS THE WHOLE POINT. Without it this is an error-handler, and the - failure it must catch is not an error: a vendor answering 200 with the one field you needed - left blank. That is exactly how Bright Data behaves on view counts, and a chain that only - caught exceptions would have stopped there forever and never reached Apify. - """ - log = log or (lambda *_a: None) - ok = satisfied or (lambda r: bool(r)) - attempts, last = [], None - for provider in chain(capability): - started = time.time() - try: - result, note = work(provider) - except Exception as e: # noqa: BLE001 - # ⚠ TYPE NAME ONLY. A vendor client's str() can carry the URL, and the credential is - # one refactor away from being a query param; this line must not be what leaks it. - result, note = None, f"{type(e).__name__}" - secs = round(time.time() - started, 2) - if note: - attempts.append(Attempt(provider.key, False, note, 0, secs)) - log(f" {provider.label}: {note}, falling through") - continue - if not ok(result): - attempts.append(Attempt(provider.key, False, "answered without the field asked for", - 0, secs)) - log(f" {provider.label}: answered, but not with what was asked for, falling through") - last = result if last is None else last - continue - attempts.append(Attempt(provider.key, True, "", _count(result), secs)) - log(f" {provider.label}: ok ({_count(result)} records, {secs}s)") - return result, attempts - return last, attempts - - -def _count(result): - if isinstance(result, (list, tuple, set)): - return len(result) - if isinstance(result, dict): - return len(result) - return 1 if result else 0 - - -def wire(): - """What the Settings surface renders. Booleans and labels — NEVER a credential. - - Mirrors the `hikerReady` rule the rest of the product follows: a surface is told WHETHER a - provider is usable, never what the key is. - """ - return { - "providers": [ - {"key": p.key, "label": p.label, "configured": p.configured(), - "capabilities": sorted(k for k, c in p.caps.items() if c.capable)} - for p in PROVIDERS.values() - ], - "chains": {cap: [p.key for p in chain(cap)] for cap in DEFAULT_CHAINS}, - "incapable": { - f"{p.key}:{cap}": c.note - for p in PROVIDERS.values() for cap, c in p.caps.items() if not c.capable - }, - } - - - - -# ═══════════════════ WAVE 36 · W36-T35 (ruling R4, contract C5) — THE LLM LADDER ════════════════ -# -# Owner item 4, verbatim (2026-08-18): *"I'm also getting errors everywhere when I want to use the -# assisntant: 'the assistant could not be reached just now (openrouter: HTTP 402)' and 'cerebras: -# HTTP 402; groq: HTTP 404; openrouter: HTTP 402; anthropic: tool calls are not wired for this -# shape'."* -# -# ⭐⭐ R4 IS THREE CLAUSES AND THEY LAND IN THREE DIFFERENT PLACES. (1) Anthropic becomes the -# tool-calling path that always works — a WIRE, in `routes_query`. (2) A provider with no credit is -# SKIPPED rather than tried — a memo, `mark_no_credit` below. (3) No raw `HTTP 402` ever reaches a -# screen — a SENTENCE, `refusal_sentence` below. The declaration here is what the first two read. -# -# ⚠ WHY A SECOND REGISTRY RATHER THAN ROWS IN `PROVIDERS` ABOVE. A scraping provider is -# `(key_env, caps)` and is billed per RECORD; an LLM provider is `(env, url, model, wire)` and is -# billed per TOKEN. Folding them into one dict would mean four fields that are meaningless for half -# the rows and an `estimate()` that answers $0.00 for anything LLM-shaped. What they SHARE is the -# thing worth sharing: `Capability`, so "MEASURED INCAPABLE" means exactly the same thing on both -# sides, and `llm_chain()` refuses an incapable row exactly as `chain()` does. -# -# ⛔ THE DECLARATION IS THE POINT (staged item 3). `_FAILED_GEN` in `routes_query` is a REGEX that -# recovers a tool call out of a provider's 400 — the evidence that guessing at capability failed. -# A row that says `llm_tool_calling: Capability(False, …)` is never offered for a tool-calling -# turn at all, so the guess never has to be made. - -#: How long a provider stays skipped after it tells us it is out of credit. ⚠ A MEMO, NOT A FACT: -#: the balance can be topped up at any moment, so this expires rather than latching. Fifteen -#: minutes is long enough that a chat session does not re-pay the timeout on every turn, and short -#: enough that a top-up is picked up without a restart. -CREDIT_COOLDOWN_S = float(os.environ.get("AIOS_CREDIT_COOLDOWN_S") or 900) - -#: `{provider name: unix ts when the memo expires}`. ⚠ PROCESS-LOCAL AND DELIBERATELY SO — it is a -#: latency optimisation, not a billing record. A second container learns the same thing from its -#: own first 402, and neither one can be wrong for longer than the cooldown. -_NO_CREDIT: dict[str, float] = {} - - -@dataclass(frozen=True) -class LlmProvider: - """One chat-completions endpoint, and what it is DECLARED able to do.""" - name: str - label: str - env: str - url: str - model: str - #: `openai` = the OpenAI-compatible `/chat/completions` shape. `anthropic` = the Messages API, - #: which is a different body, a different auth header and a different result shape. - wire: str - caps: dict = field(default_factory=dict) - - def configured(self) -> bool: - return bool((os.environ.get(self.env) or "").strip()) - - def can(self, capability: str) -> bool: - cap = self.caps.get(capability) - return bool(cap and cap.capable and self.configured()) - - -#: ⭐ ORDER IS THE LADDER, AND ANTHROPIC IS FIRST BECAUSE OF R4. `routes_query`'s old comment put -#: cerebras first *"because this path needs tool calling and cerebras carries this account's -#: tool-capable model"* — R4 replaces that premise: Anthropic is the tool-calling path that always -#: works, and the others are the cheap seats it falls through to. -LLM_PROVIDERS: dict[str, LlmProvider] = { - "anthropic": LlmProvider( - name="anthropic", label="Anthropic", env="ANTHROPIC_API_KEY", - url="https://api.anthropic.com/v1/messages", - # ⚠ Claude Opus 5, $5 / $25 per million tokens (2026-06 list). Override per deployment with - # `AIOS_ANTHROPIC_MODEL` — the id is read at call time, so a cheaper tier - # (`claude-sonnet-5`, $3 / $15) is an environment change, not a release. - # ⭐ W37-T39 re-verified this default against the vendor rather than the docs: a real - # Messages POST answers HTTP 200, and `claude-opus-5` is in `GET /v1/models`. - model=os.environ.get("AIOS_ANTHROPIC_MODEL") or "claude-opus-5", - wire="anthropic", - caps={ - # ⭐ MEASURED, and it is the whole of R4's first clause: the Messages API answers with a - # typed `tool_use` content block carrying parsed `input`. There is nothing to recover - # out of a 400 and no regex in the path — which is exactly what `_FAILED_GEN` exists to - # apologise for on the other wire. - "llm_tool_calling": Capability(True, 0.0, - "typed tool_use content block; no text recovery path"), - "llm_chat": Capability(True, 0.0, "Messages API"), - "llm_json_mode": Capability(True, 0.0, "output_config.format, schema-constrained"), - }), - "cerebras": LlmProvider( - name="cerebras", label="Cerebras", env="CEREBRAS_API_KEY", - url="https://api.cerebras.ai/v1/chat/completions", - # ⚠ STILL A LIVE ID, and W37-T39 checked rather than assumed: `gpt-oss-120b` is one of the - # two ids `GET https://api.cerebras.ai/v1/models` returns. What it is NOT is callable on - # this account — a real POST answers `HTTP 402 payment_required`, which `mark_no_credit` - # already knows how to survive (R4: skipped, not tried). - model="gpt-oss-120b", wire="openai", - caps={ - # ⚠ STILL "DECLARED, NOT MEASURED", DELIBERATELY. The 2026-08-19 sweep could not - # measure this rung: a 402 comes back before any tool is considered, so there is no - # observation to record. Leaving the old wording is the honest answer — upgrading it - # to MEASURED beside its two neighbours would be claiming an account balance as - # evidence about a capability. - "llm_tool_calling": Capability(True, 0.0, - "DECLARED, not measured: this account's tool-capable " - "model per the wave-32 ladder note. ⚠ 2026-08-19: the " - "account answers 402, so this stays unmeasured"), - "llm_chat": Capability(True, 0.0, "OpenAI-compatible"), - "llm_json_mode": Capability(True, 0.0, "response_format json_object"), - }), - "groq": LlmProvider( - name="groq", label="Groq", env="GROQ_API_KEY", - # ⛔⛔ D-345, FIXED W37-T39 (2026-08-19): this read `llama-3.3-70b-versatile` and Groq - # RETIRED it. Measured, not inferred — a real POST with the live key answered - # `HTTP 404 {"code":"model_not_found"}`, and `GET /openai/v1/models` returns 13 ids with - # that one absent. ⚠ THE STATUS IS THE EVIDENCE AND 404 IS THE ONLY ONE THAT LICENSES A - # RENAME: cerebras answered 402 on the same sweep, which is a BILLING fact about the - # account and says nothing about its model id (`gpt-oss-120b` is in its /models list). - # "Fixing" an id behind a 401/402 is how a second bug ships behind a green probe. - # ⚠ THE PREFIX IS NOT A TYPO: Groq serves this model as `openai/gpt-oss-120b` while - # Cerebras serves the same family bare as `gpt-oss-120b`. The two rungs are DELIBERATELY - # spelled differently and a sweep that "normalises" them breaks one of them. - url="https://api.groq.com/openai/v1/chat/completions", - model="openai/gpt-oss-120b", wire="openai", - caps={ - # ⭐ MEASURED 2026-08-19, W37-T39, and this comment used to say the opposite. The old - # text read *"DECLARED, not measured … `routes_query._FAILED_GEN` exists because SOME - # provider on this wire answers 400 with the call in `failed_generation`; the repo - # never recorded which. Flip this to False the day it is"*. It is recorded now: on - # this id Groq answers HTTP 200 with a TYPED `tool_calls` block carrying parsed - # `arguments`, so it is not the provider `_FAILED_GEN` apologises for. - "llm_tool_calling": Capability(True, 0.0, - "MEASURED 2026-08-19: HTTP 200, typed tool_calls block " - "with parsed arguments; no failed_generation recovery"), - "llm_chat": Capability(True, 0.0, "OpenAI-compatible"), - "llm_json_mode": Capability(True, 0.0, "response_format json_object"), - }), - "openrouter": LlmProvider( - name="openrouter", label="OpenRouter", env="OPENROUTER_API_KEY", - url="https://openrouter.ai/api/v1/chat/completions", - model="openai/gpt-4o-mini", wire="openai", - caps={ - # ⭐ MEASURED 2026-08-19, W37-T39 (was "DECLARED, not measured"): HTTP 200 with a typed - # `tool_calls` block. So on the OpenAI wire BOTH reachable rungs tool-call cleanly, and - # `_FAILED_GEN` is still holding a door nobody on this ladder has been seen to use. - "llm_tool_calling": Capability(True, 0.0, - "MEASURED 2026-08-19: HTTP 200, typed tool_calls block"), - "llm_chat": Capability(True, 0.0, "OpenAI-compatible"), - "llm_json_mode": Capability(True, 0.0, "response_format json_object"), - }), -} - -LLM_DEFAULT_ORDER = ("anthropic", "cerebras", "groq", "openrouter") - - -def mark_no_credit(name, seconds=None): - """Remember that `name` said it is out of credit, so the next turn SKIPS it (R4). - - ⛔ THE SECOND CLAUSE OF R4 IS "SKIPPED, NOT TRIED", and without a memo there is nowhere for - that to live: a stateless ladder re-tries the empty account on every single turn, pays its - round trip, and shows the reader a longer error each time. This is that memo. - """ - _NO_CREDIT[str(name)] = time.time() + float( - CREDIT_COOLDOWN_S if seconds is None else seconds) - return _NO_CREDIT[str(name)] - - -def no_credit(name): - """Is this provider inside its out-of-credit cooldown? Expiry is checked, never assumed.""" - until = _NO_CREDIT.get(str(name)) - if not until: - return False - if time.time() >= until: - _NO_CREDIT.pop(str(name), None) - return False - return True - - -def clear_credit_memo(name=None): - """Forget one memo, or all of them. For a gate, and for an operator after a top-up.""" - if name is None: - _NO_CREDIT.clear() - else: - _NO_CREDIT.pop(str(name), None) - - -def llm_chain(capability="llm_tool_calling"): - """The provider order for `capability` — declaration first, credit memo second. - - Three filters, in this order, and each removes a DIFFERENT kind of row: - 1. `can()` — declared capable AND configured. An incapable row is never offered, so a - turn cannot be spent discovering it (the `chain()` rule, one layer up). - 2. `no_credit()` — R4's skip. A provider that told us its balance is empty is passed over - until the memo expires. - 3. the ORDER itself, overridable with `AIOS_LLM_ORDER` (`anthropic,groq`) so a deployment - can be moved off a vendor without a release — the same clause `AIOS_PROVIDER_ORDER` - carries for the scraping side. - """ - raw = (os.environ.get("AIOS_LLM_ORDER") or "").strip() - names = [x.strip() for x in raw.split(",") if x.strip()] or list(LLM_DEFAULT_ORDER) - return [LLM_PROVIDERS[n] for n in names - if n in LLM_PROVIDERS and LLM_PROVIDERS[n].can(capability) and not no_credit(n)] - - -#: What an HTTP status MEANS, in words a person can act on. ⛔⛔ R4's THIRD CLAUSE LIVES HERE AND -#: IT IS NOT COSMETIC: `HTTP 402` on a screen tells a reader nothing they can do, and the owner -#: quoted it back at us twice. Every sentence names the VENDOR and the ACTION. -_STATUS_WORDS = { - 401: "{label} would not accept our key", - 403: "{label} would not accept our key", - 402: "{label} is out of credit", - 404: "{label} does not offer the model we asked it for", - 408: "{label} took too long", - 413: "the question was too long for {label}", - 429: "{label} is rate limiting us right now", -} - -#: Substrings that mean "no money" on a wire that does not use 402. ⚠ Anthropic answers a spent -#: balance with a 400 or 403 carrying a message, not with a status code of its own, so this is the -#: one place a body has to be read. It is a LOWERCASE substring test on the vendor's own words and -#: it only ever decides whether to SKIP a provider, never whether to trust one. -_CREDIT_WORDS = ("credit balance", "insufficient credit", "insufficient_quota", "out of credit", - "quota exceeded", "billing", "payment required", "add credits") - - -def is_credit_failure(status, body=""): - """Did this response mean "the account is empty"? Status first, then the vendor's own words.""" - if int(status or 0) == 402: - return True - if int(status or 0) not in (400, 403, 429): - return False - return any(word in str(body or "").lower() for word in _CREDIT_WORDS) - - -def refusal_sentence(name, status, body=""): - """One provider's failure, as a SENTENCE. Never a bare status code, never a vendor stack trace. - - ⚠ THE BODY IS READ AND NEVER QUOTED. A provider's error body can carry an account id, a key - prefix or an internal trace; the only thing taken out of it is the yes/no answer to "is this a - credit problem", and what reaches the caller is this module's own wording. - """ - label = (LLM_PROVIDERS.get(str(name)) or LlmProvider(name, str(name), "", "", "", "")).label - if is_credit_failure(status, body): - return f"{label} is out of credit" - code = int(status or 0) - if code in _STATUS_WORDS: - return _STATUS_WORDS[code].format(label=label) - if 500 <= code <= 599: - return f"{label} is having trouble at their end" - return f"{label} did not answer" - - -# ═══════════ THE ANTHROPIC WIRE, ONCE (W36-T35 / ASK D-18, ruling R4) ═══════════════════════════ -# -# ⛔⛔ TWO DOORS IN THIS PRODUCT CALL ANTHROPIC AND THEY MUST NOT EACH LEARN THE MESSAGES API. -# `routes_query._call_model` (the Assistant and Query) and `ai_review.draft_flow` (the automation -# drafter) both need it, and the owner quoted an error from EACH of them in one breath: -# *"the assistant could not be reached just now (openrouter: HTTP 402)"* and *"anthropic: tool -# calls are not wired for this shape"*. Two implementations of one wire is -# [[one-question-two-normalizers]] before a line is written, so the wire lives here, beside the -# ladder that declares the rung. -# -# FOUR THINGS THE OPENAI-COMPATIBLE SHAPE GETS WRONG, each a 400 on its own: -# 1. the system prompt is a TOP-LEVEL field, not a `{"role": "system"}` message -# 2. a tool is `{name, description, input_schema}` FLAT, not nested under `function` -# 3. `temperature` and friends are REMOVED on the current model family -# 4. `tool_choice` is an OBJECT (`{"type": "auto"}` / `{"type": "any"}`), not a string -# -# ⚠ AND ONE THING THAT IS NOT A SHAPE: `effort` is model-gated. `output_config.effort` errors on -# Haiku 4.5, so it is a PARAMETER here and the caller decides — the drafter runs on haiku and omits -# it, the assistant runs on the Opus tier and sends it. - -#: The Messages API version header. A DATE that pins the WIRE FORMAT, never a model. -ANTHROPIC_VERSION = "2023-06-01" - - -def anthropic_request(*, model, key, system, messages, tools, max_tokens, - tool_choice="auto", effort=None): - """`{url, headers, json}` for one Messages API call. Pure: reads no environment, sends nothing. - - `messages` is the OpenAI-shaped list this product already builds; the `system` turns are lifted - out of it, because that is where this API wants them. `tools` is the OpenAI-shaped tool list, - re-addressed rather than re-derived, so a schema change happens in one place. - """ - system_text = "\n\n".join(str(m.get("content") or "") for m in messages - if m.get("role") == "system") - if system: - system_text = (system_text + "\n\n" + str(system)).strip() if system_text else str(system) - turns = [{"role": ("assistant" if m.get("role") == "assistant" else "user"), - "content": str(m.get("content") or "")} - for m in messages if m.get("role") != "system" and str(m.get("content") or "").strip()] - wire_tools = [{"name": t["function"]["name"], - "description": t["function"]["description"], - "input_schema": t["function"]["parameters"]} for t in (tools or [])] - body = { - "model": str(model), - "max_tokens": int(max_tokens), - "system": system_text, - "messages": turns, - "tools": wire_tools, - } - # ⛔ W37-T39: `tool_choice` IS OMITTED WHEN THERE ARE NO TOOLS, and the builder decides that - # rather than each caller. The Messages API refuses `tool_choice` beside an empty `tools` list, - # so this used to be a `req["json"].pop("tool_choice", None)` at the one call site that sends no - # tools — a rule living outside the thing it constrains, which is [[limit-with-no-enforcer]]: - # the next no-tools caller inherits a 400 that reads like a model problem, not a shape problem. - # ⚠ Safe for all three callers today, checked rather than assumed: `routes_query` and - # `ai_review.draft_flow` both pass a non-empty tool list and keep the key exactly as before. - if wire_tools: - body["tool_choice"] = {"type": "any" if tool_choice in ("required", "any") else "auto"} - if effort: - body["output_config"] = {"effort": str(effort)} - return {"url": LLM_PROVIDERS["anthropic"].url, - "headers": {"x-api-key": str(key), - "anthropic-version": ANTHROPIC_VERSION, - "content-type": "application/json"}, - "json": body} - - -#: Cached result of `import anthropic`: the module, or `False` once it is known to be absent. -#: ⚠ `None` is NOT the "absent" sentinel — a plain falsy check would re-attempt the import on every -#: call, and a failed import is not cheap. `False` says "asked and answered". -_SDK: object = None - - -def anthropic_sdk(): - """The official `anthropic` SDK, or `None` if this deployment does not carry it. - - ⭐⭐ D-346, W37-T39. THIS FUNCTION IS THE WHOLE OF THE FIX AND ALSO THE WHOLE OF ITS RISK. - `aios-web/requirements.txt` is what the Dockerfile installs and it is OUTSIDE every lane's - fence this wave, so the pin is another session's edit. A hard `import anthropic` at module - scope would therefore turn a missing line in a manifest into a container that cannot boot at - all: the API imports this module on every request path. - - So the import is LAZY and its absence is REPORTED rather than fatal — `anthropic_send` falls - back to the raw `requests` POST that has always worked, and says which transport ran. When the - pin lands the SDK path takes over with no further change. - ⛔ The fallback is a BRIDGE, not a second implementation: both transports send the SAME body - `anthropic_request()` built and return the SAME `(status, body)` pair, so `anthropic_read`, - `is_credit_failure` and `refusal_sentence` each keep exactly one normalizer - ([[one-question-two-normalizers]]). - """ - global _SDK - if _SDK is None: - try: - import anthropic as _mod # noqa: PLC0415 - deliberately lazy; see the docstring - _SDK = _mod - except Exception: - _SDK = False - return _SDK or None - - -def anthropic_send(req, timeout=None, use_sdk=None): - """POST one `anthropic_request()` dict. Returns `(status, body, transport)`. - - ⭐ `transport` is `'sdk'` or `'http'` and it is RETURNED, not logged and forgotten: a reader - who cannot tell which path ran cannot tell whether the requirements pin reached the container - [[report-the-cause-before-you-fix-it]]. ⛔ A gate must assert on THIS RETURN VALUE, never on - `ai_review.LAST_ANTHROPIC_TRANSPORT` — that global is a convenience for `GET /meta`, and a - check keyed to it passes on a stale value written by an earlier check in the same process. - - `status` is an int and `body` is the parsed JSON dict in BOTH paths, including on an error — - the SDK raises where `requests` returns, and normalising that difference here is the only - reason this function exists rather than the caller branching on transport. - - ⚠ `use_sdk=False` forces the fallback. It exists so a gate can exercise BOTH transports - without assigning to `_SDK`: a test that mutates a module global and then throws leaves every - later check in that process running on the wrong path and passing for the wrong reason. - """ - sdk = None if use_sdk is False else anthropic_sdk() - payload = dict(req.get("json") or {}) - if sdk is None: - r = requests.post(req["url"], headers=req["headers"], json=payload, - timeout=(timeout or 60)) - try: - return r.status_code, (r.json() if r.content else {}), "http" - except Exception: - return r.status_code, {"_text": (r.text or "")[:2000]}, "http" - - key = (req.get("headers") or {}).get("x-api-key") or "" - try: - client = sdk.Anthropic(api_key=key, timeout=float(timeout or 60)) - msg = client.messages.create(**payload) - # ⚠ `.model_dump()` is what makes ONE reader serve both transports: it hands back the same - # wire-shaped dict the raw POST parses out of the response body. - return 200, msg.model_dump(), "sdk" - except Exception as exc: - # ⛔ THE STATUS IS THE PRODUCT HERE. Every sentence the reader sees is chosen by - # `refusal_sentence(status)`, so an exception that loses its code turns "Anthropic is out - # of credit" into "Anthropic did not answer" — the exact regression R4 was written to end. - status = int(getattr(exc, "status_code", 0) or 0) - body = getattr(exc, "body", None) - if not isinstance(body, dict): - body = {"_text": str(exc)[:2000]} - if not status: - # No status at all = it never reached the vendor (DNS, TLS, timeout). 408 is the one - # code `_STATUS_WORDS` already words as a reachability problem rather than a refusal. - status = 408 if isinstance(exc, getattr(sdk, "APIConnectionError", ())) else 0 - return status, body, "sdk" - - -def anthropic_read(body): - """`(text, tool_input, refusal)` out of a Messages API answer. - - ⛔ `stop_reason` IS CHECKED BEFORE `content` IS READ. A safety decline answers **HTTP 200** with - `stop_reason: "refusal"` and an empty or partial `content`, so code that indexes `content[0]` - unconditionally breaks on exactly the turn a person most needs explained. - ⭐ AND THE TOOL CALL ARRIVES PARSED. `tool_use.input` is already a dict — no `json.loads`, and - no regex recovering a call out of a 400, which is what the other wire needs. - """ - body = body if isinstance(body, dict) else {} - if str(body.get("stop_reason") or "") == "refusal": - return "", None, "the assistant declined to answer that one" - blocks = [b for b in (body.get("content") or []) if isinstance(b, dict)] - text = " ".join(str(b.get("text") or "") for b in blocks if b.get("type") == "text").strip() - calls = [b for b in blocks if b.get("type") == "tool_use"] - got = calls[0].get("input") if calls else None - return text, (got if isinstance(got, dict) else None), None - - -def llm_status(capability="llm_tool_calling"): - """Per provider: configured, declared-capable, in cooldown, and WHY — contract C5's payload. - - ⭐ ONE LIST, ONE DOOR. The Assistant's model picker and the Agent chat's toggle (W36-T34) read - THIS, so a model offered in one place cannot be missing from the other, and neither can offer a - provider the ladder would refuse to call [[permitted-is-not-answerable]]. - """ - rows = [] - for name in LLM_DEFAULT_ORDER: - p = LLM_PROVIDERS[name] - cap = p.caps.get(capability) - rows.append({ - "provider": p.name, "label": p.label, "model": p.model, "wire": p.wire, - "configured": p.configured(), - "toolCalling": bool((p.caps.get("llm_tool_calling") or Capability(False)).capable), - "jsonMode": bool((p.caps.get("llm_json_mode") or Capability(False)).capable), - "capable": bool(cap and cap.capable), - "outOfCredit": no_credit(name), - "note": (cap.note if cap else ""), - }) - return rows - - -# ============================================================================================= -# THE CANONICAL SCHEMA — owner ruling 2026-08-08: -# *"standardize the schema between Bright Data and APIfy so we keep using the same pre-set -# database even if the underlying engine changes"* -# -# ⛔ THE PRESET TABLES ARE THE CONTRACT; A VENDOR IS AN IMPLEMENTATION DETAIL. `ut_ig_posts` and -# `ut_ig_post_snapshots` must not gain, lose or rename a column because a chain was reordered — a -# tenant's saved views, filters, rollups and forms all bind to these keys, and a schema that moves -# with the vendor turns a routing change into a data migration. -# -# So every provider normalises INTO the keys below and nothing reads a vendor row downstream. -# `verify_automation` asserts both normalisers emit exactly `CANONICAL_POST_KEYS` on a fixture, so -# a third provider cannot ship with a near-miss key like `viewCount` and silently write a column -# nobody declared. -# -# ⚠ ONE NAME PER MEASUREMENT, AND `views` IS THE MEASUREMENT INSTAGRAM DISPLAYS. Meta folded -# Impressions/Plays/Video Views into a single **Views** metric on 2025-04-10, so carrying both a -# `views` and a `plays` column would be modelling a distinction the platform deleted — and it is -# exactly the distinction that let an account-grain number wear the "Views" label for a month. -# ⛔ THE VENDOR KEY THAT LOOKS RIGHT IS THE WRONG ONE, ON BOTH VENDORS. Apify ships BOTH -# `videoViewCount` (10,678) and `videoPlayCount` (137,684) for one reel whose true displayed count -# is ~137K — and Bright Data's useless `views` is 10,638 for that same reel. The two vendors' junk -# fields AGREE with each other, which is precisely what makes picking by name so dangerous. -# canonical `views` <- apify `videoPlayCount` ✅ matches the grid -# canonical `views` <- apify `videoViewCount` ⛔ off by 13x -# canonical `views` <- brightdata `views` ⛔ off by 13x AND account-grain -# ============================================================================================= - -#: Every key a normalised POST row may carry. Absent > blank: a key is omitted when the provider -#: did not answer, because `upsert_rows` merges and an empty string would ERASE what an earlier -#: paid run learned. -CANONICAL_POST_KEYS = ( - "shortcode", "url", "influencer_key", "posted_at", "type", "caption", - "likes", "comments", "views", "paid_partnership", "partner", "hashtags", - "alt_text", "tagged_location", "source_payload", - # ⭐ 2026-08-09 — the three the SECOND provider answers and the first does not. Each has a - # matching `field_def` in the engine's POST_FIELDS; a key here without a column there is a - # value that normalises cleanly and is then dropped by the write door, silently. - "plays", "video_duration", "comments_disabled", -) - - -def _int_or_none(v): - """`-1` IS NOT A COUNT. Apify returns `likesCount: -1` when the creator HIDES their like - count — a real state that is not a measurement. Writing -1 would render as a negative like - count; writing 0 would claim nobody liked it. Both are lies, so the key is omitted.""" - try: - n = int(v) - except (TypeError, ValueError): - return None - return None if n < 0 else n - - -def normalize_post_apify(row): - """One Apify `instagram-scraper` item -> the canonical post row.""" - if not isinstance(row, dict): - return None - code = str(row.get("shortCode") or "").strip() - if not code: - return None - out = { - "shortcode": code, - "url": str(row.get("url") or f"https://www.instagram.com/reel/{code}/"), - "influencer_key": str(row.get("ownerUsername") or ""), - "posted_at": str(row.get("timestamp") or "").replace("T", " ")[:16], - "type": "video" if str(row.get("type") or "").lower() == "video" else - ("carousel" if row.get("childPosts") else "image"), - "caption": str(row.get("caption") or ""), - # ⭐ THE FIELD THIS WHOLE PROVIDER EXISTS FOR. `videoPlayCount`, never `videoViewCount`: - # the wrong one agreed with Bright Data's junk (10,678 vs 10,638) on a reel whose true - # count was ~137K, and `videoPlayCount` matched a browser read to the digit. - "views": _int_or_none(row.get("videoPlayCount")), - # ⭐⭐ 2026-08-09 (owner: *"Video plays (# Plays) field isn't in APIfy? i believe it is"*). - # They were right, and the `plays` COLUMN had been empty on all 816 rows because nothing - # ever wrote it — the field existed with no writer. MEASURED on two of their own reels: - # `videoPlayCount` 216,904 / 95,331 and `videoViewCount` 115,929 / 30,871. Two different - # real numbers, and we were storing only one of them. - # ⚠ `plays` TAKES THE PLAY COUNT, which is also what `views` carries today — so the two - # columns will agree until somebody decides otherwise, and that decision is the OWNER'S: - # re-sourcing `views` to `videoViewCount` would change what the ~480 rows already captured - # mean, and a column whose meaning changes halfway down is the one thing worse than a - # column with no data. Flagged rather than done. - "plays": _int_or_none(row.get("videoPlayCount")), - "likes": _int_or_none(row.get("likesCount")), - "comments": _int_or_none(row.get("commentsCount")), - # ⭐ FIELDS BRIGHT DATA DOES NOT RETURN AT ALL, kept because they are already paid for in - # this same response (owner: *"whatever APIfy has more than BD pls use it"*). - "video_duration": _int_or_none(row.get("videoDuration")), - "comments_disabled": "1" if row.get("isCommentsDisabled") else "", - "hashtags": ", ".join(str(h) for h in (row.get("hashtags") or []) if h), - "alt_text": str(row.get("alt") or ""), - "paid_partnership": "1" if row.get("paidPartnership") else "", - "partner": ", ".join(str((s or {}).get("username") or "") - for s in (row.get("sponsors") or []) if isinstance(s, dict)), - "tagged_location": str((row.get("locationName") or "")), - "source_payload": json.dumps(row, default=str)[:32_000_000], - } - return {k: v for k, v in out.items() if v not in (None, "")} - - -def normalize_profile_apify(row): - """One Apify `instagram-scraper` PROFILE item (`resultsType: "details"`) -> our snapshot shape. - - ⭐ WHY THIS EXISTS (owner report 2026-08-09: *"I need Bright Data and Apify to work correctly - in tandem"*). `PROVIDERS["apify"]` has declared `ig_profile` capable since 2026-08-08 and - `DEFAULT_CHAINS["ig_profile"]` has read `("brightdata", "apify")` — but **nothing ever ran - that chain.** `connectors_ig.pull_profile` went Bright Data -> anonymous HTML rungs and Apify - was never asked, so a profile Bright Data cannot scrape came back `blocked` while a - configured, declared-capable provider sat unused. A registry entry with no runner is a - promise the product does not keep ([[flag-shipped-without-its-writer]]). - - ⚠ THE KEYS ARE APIFY'S camelCase, and mapping them HERE is the boundary rule this module - already enforces for posts: nothing downstream may ever see a vendor-shaped key, so the - snapshot writer cannot tell which vendor answered — which is what makes the fallback - invisible to every consumer instead of a second schema. - - ⛔ OMIT, NEVER ZERO. A field Apify did not send is dropped, exactly as `normalize_post_apify` - drops a missing count: writing 0 followers would claim we measured an empty account, and the - caller's `satisfied` hook reads absence as "did not answer" and falls through. A zero would - stop the chain on a lie. - """ - if not isinstance(row, dict): - return None - # ⛔ AN ERROR ENVELOPE IS NOT A PROFILE (measured 2026-08-09). Apify answers a dead handle - # with a 200 and a well-formed item — `{"username": …, "error": "not_found", - # "errorDescription": "Post does not exist"}` — and this function used to build a "profile" - # out of it: a username, a url and a source_payload, with every measured field absent. It - # then read to the caller as a vendor that answered, so the reason was replaced by a shrug. - # Refused HERE as well as in `connectors_ig.apify_profile` on purpose: the boundary rule this - # module exists for is that nothing downstream ever sees a vendor-shaped key, and a vendor's - # ERROR shape is the one that must never become a row. - # ⚠ TRUTHINESS, NOT KEY PRESENCE — a successful item carries `error: null` (measured on - # `sriyynntt`), so `"error" in row` would refuse every good profile. - if row.get("error"): - return None - handle = str(row.get("username") or "").strip() - if not handle: - return None - out = { - "username": handle, - "full_name": str(row.get("fullName") or ""), - "bio": str(row.get("biography") or ""), - "followers": _int_or_none(row.get("followersCount")), - "following": _int_or_none(row.get("followsCount")), - "posts_count": _int_or_none(row.get("postsCount")), - "verified": "1" if row.get("verified") else "", - "external_url": str(row.get("externalUrl") or ""), - "ig_id": str(row.get("id") or ""), - "profile_url": str(row.get("url") or f"https://www.instagram.com/{handle}/"), - "business_category": str(row.get("businessCategoryName") or ""), - "is_business": "1" if row.get("isBusinessAccount") else "", - "is_private": "1" if row.get("private") else "", - "highlights_count": _int_or_none(row.get("highlightReelCount")), - "source_payload": json.dumps(row, default=str)[:32_000_000], - } - # ⚠ `followers`/`following` are ints and 0 is a LEGITIMATE value for them, so the filter - # below must not treat 0 as absent the way the post normaliser can — an account really can - # have zero followers. Only None and "" are dropped. - return {k: v for k, v in out.items() if v is not None and v != ""} - - -def canonical_gaps(row, want=("views",)): - """Which requested canonical fields this row does NOT carry — the `satisfied` input. - - Named rather than inlined because "did the vendor actually answer the question" is the whole - fallback trigger, and a chain that asks it differently in two places will drift. - """ - row = row if isinstance(row, dict) else {} - return [k for k in want if row.get(k) in (None, "")] +"""THE PROVIDER LAYER — many vendors behind one capability, chosen per FIELD and per cost. + +⭐⭐ OWNER RULING 2026-08-08: *"Let's do a combination scraper. Do Bright Data first, and if +anything fails, we use APIfy or vice versa. Make this dynamic somehow. remember to cost optimize +when we scale to thousands+ customer … make sure our code is customizable and modular."* + +⛔ THE DESIGN DECISION THAT MATTERS, AND IT IS NOT "TRY A, THEN TRY B". +A provider-level failover ("if Bright Data errors, re-run the whole thing on Apify") is the obvious +shape and it is the expensive one. MEASURED: Bright Data answers profile, likes and comments +correctly and CANNOT answer view counts on any route; Apify answers view counts correctly. A +provider-level fallback would notice the missing views and re-buy the profile, the likes and the +comments from Apify as well — **paying twice for the 90% that already worked**. At one profile that +is noise; at thousands of customers × twelve posts each it is the whole bill. + +So routing is per CAPABILITY, and a capability is the smallest independently-billable thing: + + ig_profile -> brightdata (36 fields, works) + ig_post_metrics -> brightdata (likes + comments, works) + ig_post_views -> apify (Bright Data is INCAPABLE — see `capable=False` below) + ig_comments -> brightdata + +Each capability has an ORDERED list of providers. The first one that is configured AND capable AND +succeeds wins; the rest are never called, so the common path costs exactly one vendor record. + +⚠ "FAILURE" INCLUDES ANSWERING WITH THE FIELD BLANK. A vendor that returns HTTP 200 and a null in +the one column you asked for has failed for our purposes, and a chain that only catches exceptions +would stop at it forever. `run()` takes a `satisfied` predicate and falls through on an unsatisfying +answer exactly as it would on a 500. + +⚠ ORDER IS DATA, NOT CODE. `AIOS_PROVIDER_ORDER` overrides any chain at deploy time +(`ig_post_views=apify,brightdata;ig_profile=apify`), which is what "or vice versa" means and what +lets a tenant be moved off a vendor without a release. +""" +from __future__ import annotations + +import json +import os +import time +from dataclasses import dataclass, field +from typing import Callable + +# ⚠ TOP-LEVEL AND SAFE, unlike the `anthropic` import in `anthropic_sdk()`: `requests` is pinned in +# BOTH manifests and is already a transitive dependency of `huggingface_hub`, so it cannot be the +# line that stops a container booting. That asymmetry is the whole reason the two are imported +# differently — see `anthropic_sdk()`. +import requests + +#: ⚠ APPROXIMATE, AND DELIBERATELY SO. These are list prices per RECORD in USD, used only to RANK +#: providers and to estimate a run's spend for the operator. They are not billing truth — the +#: vendor's own dashboard is. Wrong by 2x still ranks correctly; wrong by 100x does not, which is +#: why they are named and dated rather than guessed silently. +#: Bright Data Instagram datasets ~ $0.0015/record (2026-08 list). Apify instagram-scraper +#: ~ $0.0027/result (2026-08 list). Both re-checked when a provider is added. +#: ⚠ WAVE 30 · T16 — APIFY CORRECTED 0.0023 -> 0.0027, and the direction matters: the old number +#: made the fallback look CHEAPER than it is, and `estimate()` is what a person is shown before +#: they authorise a run. An under-stated price is the one rounding error a cost guard cannot catch. +#: ⚠ Both are LIST rates used to rank and to estimate. They are not invoices, and nothing here +#: reads a live price — a modelled figure that says so is honest; one that pretends is not. +_COST_BRIGHTDATA = float(os.environ.get("AIOS_COST_BRIGHTDATA") or 0.0015) +_COST_APIFY = float(os.environ.get("AIOS_COST_APIFY") or 0.0027) + + +@dataclass(frozen=True) +class Capability: + """What ONE provider can do for ONE capability, and what it costs to ask.""" + #: ⛔ `False` means MEASURED INCAPABLE, not "untried". A provider declared incapable is never + #: called for this capability at all — it cannot be reached by a fallback, cannot be put first + #: by an env override, and cannot silently start being billed because somebody reordered a + #: list. Bright Data's `ig_post_views` is False on the strength of §4e-§4h: every route + #: (Posts x /p/, Posts x /reel/, Reels x /p/, Reels x /reel/, discover-by-profile) on four + #: accounts from 13K to 268M followers returned an account-grain constant or a null. + capable: bool = True + cost_per_record: float = 0.0 + #: Free-text, shown to an operator deciding where their money went. + note: str = "" + + +@dataclass +class Provider: + key: str + label: str + #: The env var holding this provider's credential. NEVER the credential itself — this module + #: is imported by surfaces that serialise their config. + key_env: str = "" + caps: dict = field(default_factory=dict) + + def configured(self) -> bool: + return bool((os.environ.get(self.key_env) or "").strip()) if self.key_env else True + + def cap(self, capability: str) -> Capability | None: + return self.caps.get(capability) + + def can(self, capability: str) -> bool: + c = self.cap(capability) + return bool(c and c.capable and self.configured()) + + +#: ⚠ ADDING A PROVIDER IS A REGISTRY ENTRY PLUS A RUNNER — no change to any caller. That is the +#: "modular/customizable" half of the ruling, and the reason the chains below name STRINGS. +PROVIDERS: dict[str, Provider] = { + "brightdata": Provider( + key="brightdata", label="Bright Data", key_env="AIOS_BRIGHTDATA_KEY", + caps={ + "ig_profile": Capability(True, _COST_BRIGHTDATA, "36 fields incl. bio/email/category"), + "ig_post_metrics": Capability(True, _COST_BRIGHTDATA, "likes + comments + captions"), + "ig_comments": Capability(True, _COST_BRIGHTDATA, "separate paid dataset, opt-in"), + # ⛔ THE ONE THAT COSTS US NOTHING TO GET RIGHT AND EVERYTHING TO GET WRONG. + "ig_post_views": Capability( + False, _COST_BRIGHTDATA, + "MEASURED INCAPABLE: returns one account-grain number for every reel of a " + "creator (identical for two shortcodes in one call, ticking upward between " + "calls) and never populates video_play_count. Their own documented example " + "account no longer reproduces it."), + # ⭐⭐ WAVE 29 (item 7 / D-9 / R1) — TIKTOK. Three datasets, all reachable with our + # existing key, all schema-probed for $0.00 (40/43/17 fields with the vendor's own + # types): `gd_l1villgoiiidt09ci` · `gd_lu702nij2f790tmv9h` · `gd_lkf2st302ap89utw5k`. + "tt_profile": Capability(True, _COST_BRIGHTDATA, + "40 fields incl. bio/engagement rates/region"), + # ⛔ THERE IS NO `tt_post_views` CAPABILITY, AND ITS ABSENCE IS THE CLAIM. + # `TikTok - Posts` declares `play_count: number` and the vendor's own sample says + # "no empty values or zeros" — so on TikTok the view count arrives INSIDE the post + # record and a separate view rung would be a second bill for a number we already have. + # ⚠ DECLARED, NOT MEASURED, and that distinction is the whole scar tissue of §4e: + # Bright Data's Instagram Reels also DECLARES `views: number` and delivers an + # account-grain wrong one. One live pull settles it (W29-T06, spend-gated). If it + # fails, the honest repair is a `tt_post_views` capability routed elsewhere — never a + # quiet fallback bolted onto this one. + "tt_post_metrics": Capability( + True, _COST_BRIGHTDATA, + "likes + comments + shares + saves, and play_count inline (DECLARED " + "no-empties-or-zeros, UNPROVEN until one live pull)"), + "tt_comments": Capability(True, _COST_BRIGHTDATA, + "separate paid dataset, opt-in: 17 fields"), + }), + "apify": Provider( + key="apify", label="Apify", key_env="AIOS_APIFY_KEY", + caps={ + # ⭐ MEASURED 2026-08-08 against the public Reels grid: `videoPlayCount` 137,684 and + # 299,493 vs a browser-read ground truth of 134K-137K and 299K. Exact. + "ig_post_views": Capability(True, _COST_APIFY, + "videoPlayCount, matches Instagram's displayed views"), + "ig_post_metrics": Capability(True, _COST_APIFY, "likes + comments (fallback)"), + "ig_profile": Capability(True, _COST_APIFY, "profile fields (fallback)"), + }), +} + +#: The DEFAULT chain per capability, cheapest-capable-first. Overridable — see `chain()`. +DEFAULT_CHAINS: dict[str, tuple] = { + "ig_profile": ("brightdata", "apify"), + "ig_post_metrics": ("brightdata", "apify"), + # Only one entry, and that is the point: Bright Data is declared incapable, so listing it here + # would be a lie that costs a wasted call on every single post. + "ig_post_views": ("apify",), + "ig_comments": ("brightdata",), + # ⭐ WAVE 29 — TikTok, and every chain is deliberately SINGLE-PROVIDER. + # ⛔ A MULTI-PROVIDER CHAIN IS A PROMISE SOMETHING WALKS IT. `ig_post_metrics` has declared a + # two-provider fallback since wave 28 and NOTHING reaches the second name: if Bright Data + # answers with the likes blank, Apify is never asked. `verify_automation`'s E2b pins that gap + # BY NAME so a third unwalked chain turns it red — which is exactly what a second name here + # would be today. Apify does sell TikTok (`clockworks/tiktok-profile-scraper`, $0.003/result, + # 0.7% 30-day failure rate — measured from its own store record), so the fallback is buildable; + # it is not declared until a runner walks it. + "tt_profile": ("brightdata",), + "tt_post_metrics": ("brightdata",), + "tt_comments": ("brightdata",), +} + + +def chain(capability: str) -> list: + """The provider order for `capability` — env override first, then the default. + + `AIOS_PROVIDER_ORDER` is a `;`-separated list of `capability=p1,p2` clauses. This is how the + owner's "or vice versa" is expressed WITHOUT a release, and how one tenant can be moved off a + vendor that is having a bad day. + ⛔ An override may REORDER and may DROP, but it can never make an incapable provider capable — + `can()` still gates every name. A chain that reads `ig_post_views=brightdata` therefore + resolves to EMPTY rather than to a provider that would return a wrong number, because a + plausible wrong number is worse than an honest refusal. + """ + raw = (os.environ.get("AIOS_PROVIDER_ORDER") or "").strip() + names = None + for clause in raw.split(";"): + if "=" in clause: + cap_name, _, order = clause.partition("=") + if cap_name.strip() == capability: + names = [x.strip() for x in order.split(",") if x.strip()] + if names is None: + names = list(DEFAULT_CHAINS.get(capability) or ()) + return [PROVIDERS[n] for n in names if n in PROVIDERS and PROVIDERS[n].can(capability)] + + +def estimate(capability: str, records: int) -> dict: + """What the FIRST capable provider would cost for `records` — the number an operator plans on. + + Reported per capability rather than per run because that is the unit that scales: at a thousand + tenants the question is never "what did this run cost", it is "what does adding view counts to + every post cost per month". + """ + ch = chain(capability) + if not ch: + return {"capability": capability, "records": records, "provider": None, "usd": 0.0, + "note": "no configured, capable provider"} + p = ch[0] + c = p.cap(capability) + return {"capability": capability, "records": records, "provider": p.key, + "usd": round((c.cost_per_record or 0.0) * max(0, int(records)), 4), + "note": c.note} + + +@dataclass +class Attempt: + provider: str + ok: bool + note: str = "" + records: int = 0 + seconds: float = 0.0 + + +def run(capability, work, satisfied=None, log=None): + """Walk the chain for `capability` until one provider gives a SATISFYING answer. + + `work(provider) -> (result, note)`; a non-empty note means it did not answer. + `satisfied(result) -> bool` decides whether an answer is good enough to stop. Default: any + truthy result. + + Returns `(result, attempts)`. `attempts` is the audit trail — every provider tried, whether it + satisfied, and how long it took — because "where did this number come from and what did it + cost" is a question somebody asks about a bill, not about a stack trace. + + ⛔ THE `satisfied` HOOK IS THE WHOLE POINT. Without it this is an error-handler, and the + failure it must catch is not an error: a vendor answering 200 with the one field you needed + left blank. That is exactly how Bright Data behaves on view counts, and a chain that only + caught exceptions would have stopped there forever and never reached Apify. + """ + log = log or (lambda *_a: None) + ok = satisfied or (lambda r: bool(r)) + attempts, last = [], None + for provider in chain(capability): + started = time.time() + try: + result, note = work(provider) + except Exception as e: # noqa: BLE001 + # ⚠ TYPE NAME ONLY. A vendor client's str() can carry the URL, and the credential is + # one refactor away from being a query param; this line must not be what leaks it. + result, note = None, f"{type(e).__name__}" + secs = round(time.time() - started, 2) + if note: + attempts.append(Attempt(provider.key, False, note, 0, secs)) + log(f" {provider.label}: {note}, falling through") + continue + if not ok(result): + attempts.append(Attempt(provider.key, False, "answered without the field asked for", + 0, secs)) + log(f" {provider.label}: answered, but not with what was asked for, falling through") + last = result if last is None else last + continue + attempts.append(Attempt(provider.key, True, "", _count(result), secs)) + log(f" {provider.label}: ok ({_count(result)} records, {secs}s)") + return result, attempts + return last, attempts + + +def _count(result): + if isinstance(result, (list, tuple, set)): + return len(result) + if isinstance(result, dict): + return len(result) + return 1 if result else 0 + + +def wire(): + """What the Settings surface renders. Booleans and labels — NEVER a credential. + + Mirrors the `hikerReady` rule the rest of the product follows: a surface is told WHETHER a + provider is usable, never what the key is. + """ + return { + "providers": [ + {"key": p.key, "label": p.label, "configured": p.configured(), + "capabilities": sorted(k for k, c in p.caps.items() if c.capable)} + for p in PROVIDERS.values() + ], + "chains": {cap: [p.key for p in chain(cap)] for cap in DEFAULT_CHAINS}, + "incapable": { + f"{p.key}:{cap}": c.note + for p in PROVIDERS.values() for cap, c in p.caps.items() if not c.capable + }, + } + + + + +# ═══════════════════ WAVE 36 · W36-T35 (ruling R4, contract C5) — THE LLM LADDER ════════════════ +# +# Owner item 4, verbatim (2026-08-18): *"I'm also getting errors everywhere when I want to use the +# assisntant: 'the assistant could not be reached just now (openrouter: HTTP 402)' and 'cerebras: +# HTTP 402; groq: HTTP 404; openrouter: HTTP 402; anthropic: tool calls are not wired for this +# shape'."* +# +# ⭐⭐ R4 IS THREE CLAUSES AND THEY LAND IN THREE DIFFERENT PLACES. (1) Anthropic becomes the +# tool-calling path that always works — a WIRE, in `routes_query`. (2) A provider with no credit is +# SKIPPED rather than tried — a memo, `mark_no_credit` below. (3) No raw `HTTP 402` ever reaches a +# screen — a SENTENCE, `refusal_sentence` below. The declaration here is what the first two read. +# +# ⚠ WHY A SECOND REGISTRY RATHER THAN ROWS IN `PROVIDERS` ABOVE. A scraping provider is +# `(key_env, caps)` and is billed per RECORD; an LLM provider is `(env, url, model, wire)` and is +# billed per TOKEN. Folding them into one dict would mean four fields that are meaningless for half +# the rows and an `estimate()` that answers $0.00 for anything LLM-shaped. What they SHARE is the +# thing worth sharing: `Capability`, so "MEASURED INCAPABLE" means exactly the same thing on both +# sides, and `llm_chain()` refuses an incapable row exactly as `chain()` does. +# +# ⛔ THE DECLARATION IS THE POINT (staged item 3). `_FAILED_GEN` in `routes_query` is a REGEX that +# recovers a tool call out of a provider's 400 — the evidence that guessing at capability failed. +# A row that says `llm_tool_calling: Capability(False, …)` is never offered for a tool-calling +# turn at all, so the guess never has to be made. + +#: How long a provider stays skipped after it tells us it is out of credit. ⚠ A MEMO, NOT A FACT: +#: the balance can be topped up at any moment, so this expires rather than latching. Fifteen +#: minutes is long enough that a chat session does not re-pay the timeout on every turn, and short +#: enough that a top-up is picked up without a restart. +CREDIT_COOLDOWN_S = float(os.environ.get("AIOS_CREDIT_COOLDOWN_S") or 900) + +#: `{provider name: unix ts when the memo expires}`. ⚠ PROCESS-LOCAL AND DELIBERATELY SO — it is a +#: latency optimisation, not a billing record. A second container learns the same thing from its +#: own first 402, and neither one can be wrong for longer than the cooldown. +_NO_CREDIT: dict[str, float] = {} + + +@dataclass(frozen=True) +class LlmProvider: + """One chat-completions endpoint, and what it is DECLARED able to do.""" + name: str + label: str + env: str + url: str + model: str + #: `openai` = the OpenAI-compatible `/chat/completions` shape. `anthropic` = the Messages API, + #: which is a different body, a different auth header and a different result shape. + wire: str + caps: dict = field(default_factory=dict) + + def configured(self) -> bool: + return bool((os.environ.get(self.env) or "").strip()) + + def can(self, capability: str) -> bool: + cap = self.caps.get(capability) + return bool(cap and cap.capable and self.configured()) + + +#: ⭐ ORDER IS THE LADDER, AND ANTHROPIC IS FIRST BECAUSE OF R4. `routes_query`'s old comment put +#: cerebras first *"because this path needs tool calling and cerebras carries this account's +#: tool-capable model"* — R4 replaces that premise: Anthropic is the tool-calling path that always +#: works, and the others are the cheap seats it falls through to. +LLM_PROVIDERS: dict[str, LlmProvider] = { + "anthropic": LlmProvider( + name="anthropic", label="Anthropic", env="ANTHROPIC_API_KEY", + url="https://api.anthropic.com/v1/messages", + # ⚠ Claude Opus 5, $5 / $25 per million tokens (2026-06 list). Override per deployment with + # `AIOS_ANTHROPIC_MODEL` — the id is read at call time, so a cheaper tier + # (`claude-sonnet-5`, $3 / $15) is an environment change, not a release. + # ⭐ W37-T39 re-verified this default against the vendor rather than the docs: a real + # Messages POST answers HTTP 200, and `claude-opus-5` is in `GET /v1/models`. + model=os.environ.get("AIOS_ANTHROPIC_MODEL") or "claude-opus-5", + wire="anthropic", + caps={ + # ⭐ MEASURED, and it is the whole of R4's first clause: the Messages API answers with a + # typed `tool_use` content block carrying parsed `input`. There is nothing to recover + # out of a 400 and no regex in the path — which is exactly what `_FAILED_GEN` exists to + # apologise for on the other wire. + "llm_tool_calling": Capability(True, 0.0, + "typed tool_use content block; no text recovery path"), + "llm_chat": Capability(True, 0.0, "Messages API"), + "llm_json_mode": Capability(True, 0.0, "output_config.format, schema-constrained"), + }), + "cerebras": LlmProvider( + name="cerebras", label="Cerebras", env="CEREBRAS_API_KEY", + url="https://api.cerebras.ai/v1/chat/completions", + # ⚠ STILL A LIVE ID, and W37-T39 checked rather than assumed: `gpt-oss-120b` is one of the + # two ids `GET https://api.cerebras.ai/v1/models` returns. What it is NOT is callable on + # this account — a real POST answers `HTTP 402 payment_required`, which `mark_no_credit` + # already knows how to survive (R4: skipped, not tried). + model="gpt-oss-120b", wire="openai", + caps={ + # ⚠ STILL "DECLARED, NOT MEASURED", DELIBERATELY. The 2026-08-19 sweep could not + # measure this rung: a 402 comes back before any tool is considered, so there is no + # observation to record. Leaving the old wording is the honest answer — upgrading it + # to MEASURED beside its two neighbours would be claiming an account balance as + # evidence about a capability. + "llm_tool_calling": Capability(True, 0.0, + "DECLARED, not measured: this account's tool-capable " + "model per the wave-32 ladder note. ⚠ 2026-08-19: the " + "account answers 402, so this stays unmeasured"), + "llm_chat": Capability(True, 0.0, "OpenAI-compatible"), + "llm_json_mode": Capability(True, 0.0, "response_format json_object"), + }), + "groq": LlmProvider( + name="groq", label="Groq", env="GROQ_API_KEY", + # ⛔⛔ D-345, FIXED W37-T39 (2026-08-19): this read `llama-3.3-70b-versatile` and Groq + # RETIRED it. Measured, not inferred — a real POST with the live key answered + # `HTTP 404 {"code":"model_not_found"}`, and `GET /openai/v1/models` returns 13 ids with + # that one absent. ⚠ THE STATUS IS THE EVIDENCE AND 404 IS THE ONLY ONE THAT LICENSES A + # RENAME: cerebras answered 402 on the same sweep, which is a BILLING fact about the + # account and says nothing about its model id (`gpt-oss-120b` is in its /models list). + # "Fixing" an id behind a 401/402 is how a second bug ships behind a green probe. + # ⚠ THE PREFIX IS NOT A TYPO: Groq serves this model as `openai/gpt-oss-120b` while + # Cerebras serves the same family bare as `gpt-oss-120b`. The two rungs are DELIBERATELY + # spelled differently and a sweep that "normalises" them breaks one of them. + url="https://api.groq.com/openai/v1/chat/completions", + model="openai/gpt-oss-120b", wire="openai", + caps={ + # ⭐ MEASURED 2026-08-19, W37-T39, and this comment used to say the opposite. The old + # text read *"DECLARED, not measured … `routes_query._FAILED_GEN` exists because SOME + # provider on this wire answers 400 with the call in `failed_generation`; the repo + # never recorded which. Flip this to False the day it is"*. It is recorded now: on + # this id Groq answers HTTP 200 with a TYPED `tool_calls` block carrying parsed + # `arguments`, so it is not the provider `_FAILED_GEN` apologises for. + "llm_tool_calling": Capability(True, 0.0, + "MEASURED 2026-08-19: HTTP 200, typed tool_calls block " + "with parsed arguments; no failed_generation recovery"), + "llm_chat": Capability(True, 0.0, "OpenAI-compatible"), + "llm_json_mode": Capability(True, 0.0, "response_format json_object"), + }), + "openrouter": LlmProvider( + name="openrouter", label="OpenRouter", env="OPENROUTER_API_KEY", + url="https://openrouter.ai/api/v1/chat/completions", + model="openai/gpt-4o-mini", wire="openai", + caps={ + # ⭐ MEASURED 2026-08-19, W37-T39 (was "DECLARED, not measured"): HTTP 200 with a typed + # `tool_calls` block. So on the OpenAI wire BOTH reachable rungs tool-call cleanly, and + # `_FAILED_GEN` is still holding a door nobody on this ladder has been seen to use. + "llm_tool_calling": Capability(True, 0.0, + "MEASURED 2026-08-19: HTTP 200, typed tool_calls block"), + "llm_chat": Capability(True, 0.0, "OpenAI-compatible"), + "llm_json_mode": Capability(True, 0.0, "response_format json_object"), + }), +} + +LLM_DEFAULT_ORDER = ("anthropic", "cerebras", "groq", "openrouter") + + +def mark_no_credit(name, seconds=None): + """Remember that `name` said it is out of credit, so the next turn SKIPS it (R4). + + ⛔ THE SECOND CLAUSE OF R4 IS "SKIPPED, NOT TRIED", and without a memo there is nowhere for + that to live: a stateless ladder re-tries the empty account on every single turn, pays its + round trip, and shows the reader a longer error each time. This is that memo. + """ + _NO_CREDIT[str(name)] = time.time() + float( + CREDIT_COOLDOWN_S if seconds is None else seconds) + return _NO_CREDIT[str(name)] + + +def no_credit(name): + """Is this provider inside its out-of-credit cooldown? Expiry is checked, never assumed.""" + until = _NO_CREDIT.get(str(name)) + if not until: + return False + if time.time() >= until: + _NO_CREDIT.pop(str(name), None) + return False + return True + + +def clear_credit_memo(name=None): + """Forget one memo, or all of them. For a gate, and for an operator after a top-up.""" + if name is None: + _NO_CREDIT.clear() + else: + _NO_CREDIT.pop(str(name), None) + + +def llm_chain(capability="llm_tool_calling"): + """The provider order for `capability` — declaration first, credit memo second. + + Three filters, in this order, and each removes a DIFFERENT kind of row: + 1. `can()` — declared capable AND configured. An incapable row is never offered, so a + turn cannot be spent discovering it (the `chain()` rule, one layer up). + 2. `no_credit()` — R4's skip. A provider that told us its balance is empty is passed over + until the memo expires. + 3. the ORDER itself, overridable with `AIOS_LLM_ORDER` (`anthropic,groq`) so a deployment + can be moved off a vendor without a release — the same clause `AIOS_PROVIDER_ORDER` + carries for the scraping side. + """ + raw = (os.environ.get("AIOS_LLM_ORDER") or "").strip() + names = [x.strip() for x in raw.split(",") if x.strip()] or list(LLM_DEFAULT_ORDER) + return [LLM_PROVIDERS[n] for n in names + if n in LLM_PROVIDERS and LLM_PROVIDERS[n].can(capability) and not no_credit(n)] + + +#: What an HTTP status MEANS, in words a person can act on. ⛔⛔ R4's THIRD CLAUSE LIVES HERE AND +#: IT IS NOT COSMETIC: `HTTP 402` on a screen tells a reader nothing they can do, and the owner +#: quoted it back at us twice. Every sentence names the VENDOR and the ACTION. +_STATUS_WORDS = { + 401: "{label} would not accept our key", + 403: "{label} would not accept our key", + 402: "{label} is out of credit", + 404: "{label} does not offer the model we asked it for", + 408: "{label} took too long", + 413: "the question was too long for {label}", + 429: "{label} is rate limiting us right now", +} + +#: Substrings that mean "no money" on a wire that does not use 402. ⚠ Anthropic answers a spent +#: balance with a 400 or 403 carrying a message, not with a status code of its own, so this is the +#: one place a body has to be read. It is a LOWERCASE substring test on the vendor's own words and +#: it only ever decides whether to SKIP a provider, never whether to trust one. +_CREDIT_WORDS = ("credit balance", "insufficient credit", "insufficient_quota", "out of credit", + "quota exceeded", "billing", "payment required", "add credits") + + +def is_credit_failure(status, body=""): + """Did this response mean "the account is empty"? Status first, then the vendor's own words.""" + if int(status or 0) == 402: + return True + if int(status or 0) not in (400, 403, 429): + return False + return any(word in str(body or "").lower() for word in _CREDIT_WORDS) + + +def refusal_sentence(name, status, body=""): + """One provider's failure, as a SENTENCE. Never a bare status code, never a vendor stack trace. + + ⚠ THE BODY IS READ AND NEVER QUOTED. A provider's error body can carry an account id, a key + prefix or an internal trace; the only thing taken out of it is the yes/no answer to "is this a + credit problem", and what reaches the caller is this module's own wording. + """ + label = (LLM_PROVIDERS.get(str(name)) or LlmProvider(name, str(name), "", "", "", "")).label + if is_credit_failure(status, body): + return f"{label} is out of credit" + code = int(status or 0) + if code in _STATUS_WORDS: + return _STATUS_WORDS[code].format(label=label) + if 500 <= code <= 599: + return f"{label} is having trouble at their end" + return f"{label} did not answer" + + +# ═══════════ THE ANTHROPIC WIRE, ONCE (W36-T35 / ASK D-18, ruling R4) ═══════════════════════════ +# +# ⛔⛔ TWO DOORS IN THIS PRODUCT CALL ANTHROPIC AND THEY MUST NOT EACH LEARN THE MESSAGES API. +# `routes_query._call_model` (the Assistant and Query) and `ai_review.draft_flow` (the automation +# drafter) both need it, and the owner quoted an error from EACH of them in one breath: +# *"the assistant could not be reached just now (openrouter: HTTP 402)"* and *"anthropic: tool +# calls are not wired for this shape"*. Two implementations of one wire is +# [[one-question-two-normalizers]] before a line is written, so the wire lives here, beside the +# ladder that declares the rung. +# +# FOUR THINGS THE OPENAI-COMPATIBLE SHAPE GETS WRONG, each a 400 on its own: +# 1. the system prompt is a TOP-LEVEL field, not a `{"role": "system"}` message +# 2. a tool is `{name, description, input_schema}` FLAT, not nested under `function` +# 3. `temperature` and friends are REMOVED on the current model family +# 4. `tool_choice` is an OBJECT (`{"type": "auto"}` / `{"type": "any"}`), not a string +# +# ⚠ AND ONE THING THAT IS NOT A SHAPE: `effort` is model-gated. `output_config.effort` errors on +# Haiku 4.5, so it is a PARAMETER here and the caller decides — the drafter runs on haiku and omits +# it, the assistant runs on the Opus tier and sends it. + +#: The Messages API version header. A DATE that pins the WIRE FORMAT, never a model. +ANTHROPIC_VERSION = "2023-06-01" + + +def anthropic_request(*, model, key, system, messages, tools, max_tokens, + tool_choice="auto", effort=None): + """`{url, headers, json}` for one Messages API call. Pure: reads no environment, sends nothing. + + `messages` is the OpenAI-shaped list this product already builds; the `system` turns are lifted + out of it, because that is where this API wants them. `tools` is the OpenAI-shaped tool list, + re-addressed rather than re-derived, so a schema change happens in one place. + """ + system_text = "\n\n".join(str(m.get("content") or "") for m in messages + if m.get("role") == "system") + if system: + system_text = (system_text + "\n\n" + str(system)).strip() if system_text else str(system) + turns = [{"role": ("assistant" if m.get("role") == "assistant" else "user"), + "content": str(m.get("content") or "")} + for m in messages if m.get("role") != "system" and str(m.get("content") or "").strip()] + wire_tools = [{"name": t["function"]["name"], + "description": t["function"]["description"], + "input_schema": t["function"]["parameters"]} for t in (tools or [])] + body = { + "model": str(model), + "max_tokens": int(max_tokens), + "system": system_text, + "messages": turns, + "tools": wire_tools, + } + # ⛔ W37-T39: `tool_choice` IS OMITTED WHEN THERE ARE NO TOOLS, and the builder decides that + # rather than each caller. The Messages API refuses `tool_choice` beside an empty `tools` list, + # so this used to be a `req["json"].pop("tool_choice", None)` at the one call site that sends no + # tools — a rule living outside the thing it constrains, which is [[limit-with-no-enforcer]]: + # the next no-tools caller inherits a 400 that reads like a model problem, not a shape problem. + # ⚠ Safe for all three callers today, checked rather than assumed: `routes_query` and + # `ai_review.draft_flow` both pass a non-empty tool list and keep the key exactly as before. + if wire_tools: + body["tool_choice"] = {"type": "any" if tool_choice in ("required", "any") else "auto"} + if effort: + body["output_config"] = {"effort": str(effort)} + return {"url": LLM_PROVIDERS["anthropic"].url, + "headers": {"x-api-key": str(key), + "anthropic-version": ANTHROPIC_VERSION, + "content-type": "application/json"}, + "json": body} + + +#: Cached result of `import anthropic`: the module, or `False` once it is known to be absent. +#: ⚠ `None` is NOT the "absent" sentinel — a plain falsy check would re-attempt the import on every +#: call, and a failed import is not cheap. `False` says "asked and answered". +_SDK: object = None + + +def anthropic_sdk(): + """The official `anthropic` SDK, or `None` if this deployment does not carry it. + + ⭐⭐ D-346, W37-T39. THIS FUNCTION IS THE WHOLE OF THE FIX AND ALSO THE WHOLE OF ITS RISK. + `aios-web/requirements.txt` is what the Dockerfile installs and it is OUTSIDE every lane's + fence this wave, so the pin is another session's edit. A hard `import anthropic` at module + scope would therefore turn a missing line in a manifest into a container that cannot boot at + all: the API imports this module on every request path. + + So the import is LAZY and its absence is REPORTED rather than fatal — `anthropic_send` falls + back to the raw `requests` POST that has always worked, and says which transport ran. When the + pin lands the SDK path takes over with no further change. + ⛔ The fallback is a BRIDGE, not a second implementation: both transports send the SAME body + `anthropic_request()` built and return the SAME `(status, body)` pair, so `anthropic_read`, + `is_credit_failure` and `refusal_sentence` each keep exactly one normalizer + ([[one-question-two-normalizers]]). + """ + global _SDK + if _SDK is None: + try: + import anthropic as _mod # noqa: PLC0415 - deliberately lazy; see the docstring + _SDK = _mod + except Exception: + _SDK = False + return _SDK or None + + +def anthropic_send(req, timeout=None, use_sdk=None): + """POST one `anthropic_request()` dict. Returns `(status, body, transport)`. + + ⭐ `transport` is `'sdk'` or `'http'` and it is RETURNED, not logged and forgotten: a reader + who cannot tell which path ran cannot tell whether the requirements pin reached the container + [[report-the-cause-before-you-fix-it]]. ⛔ A gate must assert on THIS RETURN VALUE, never on + `ai_review.LAST_ANTHROPIC_TRANSPORT` — that global is a convenience for `GET /meta`, and a + check keyed to it passes on a stale value written by an earlier check in the same process. + + `status` is an int and `body` is the parsed JSON dict in BOTH paths, including on an error — + the SDK raises where `requests` returns, and normalising that difference here is the only + reason this function exists rather than the caller branching on transport. + + ⚠ `use_sdk=False` forces the fallback. It exists so a gate can exercise BOTH transports + without assigning to `_SDK`: a test that mutates a module global and then throws leaves every + later check in that process running on the wrong path and passing for the wrong reason. + """ + sdk = None if use_sdk is False else anthropic_sdk() + payload = dict(req.get("json") or {}) + if sdk is None: + r = requests.post(req["url"], headers=req["headers"], json=payload, + timeout=(timeout or 60)) + try: + return r.status_code, (r.json() if r.content else {}), "http" + except Exception: + return r.status_code, {"_text": (r.text or "")[:2000]}, "http" + + key = (req.get("headers") or {}).get("x-api-key") or "" + try: + client = sdk.Anthropic(api_key=key, timeout=float(timeout or 60)) + msg = client.messages.create(**payload) + # ⚠ `.model_dump()` is what makes ONE reader serve both transports: it hands back the same + # wire-shaped dict the raw POST parses out of the response body. + return 200, msg.model_dump(), "sdk" + except Exception as exc: + # ⛔ THE STATUS IS THE PRODUCT HERE. Every sentence the reader sees is chosen by + # `refusal_sentence(status)`, so an exception that loses its code turns "Anthropic is out + # of credit" into "Anthropic did not answer" — the exact regression R4 was written to end. + status = int(getattr(exc, "status_code", 0) or 0) + body = getattr(exc, "body", None) + if not isinstance(body, dict): + body = {"_text": str(exc)[:2000]} + if not status: + # No status at all = it never reached the vendor (DNS, TLS, timeout). 408 is the one + # code `_STATUS_WORDS` already words as a reachability problem rather than a refusal. + status = 408 if isinstance(exc, getattr(sdk, "APIConnectionError", ())) else 0 + return status, body, "sdk" + + +def anthropic_read(body): + """`(text, tool_input, refusal)` out of a Messages API answer. + + ⛔ `stop_reason` IS CHECKED BEFORE `content` IS READ. A safety decline answers **HTTP 200** with + `stop_reason: "refusal"` and an empty or partial `content`, so code that indexes `content[0]` + unconditionally breaks on exactly the turn a person most needs explained. + ⭐ AND THE TOOL CALL ARRIVES PARSED. `tool_use.input` is already a dict — no `json.loads`, and + no regex recovering a call out of a 400, which is what the other wire needs. + """ + body = body if isinstance(body, dict) else {} + if str(body.get("stop_reason") or "") == "refusal": + return "", None, "the assistant declined to answer that one" + blocks = [b for b in (body.get("content") or []) if isinstance(b, dict)] + text = " ".join(str(b.get("text") or "") for b in blocks if b.get("type") == "text").strip() + calls = [b for b in blocks if b.get("type") == "tool_use"] + got = calls[0].get("input") if calls else None + return text, (got if isinstance(got, dict) else None), None + + +def llm_status(capability="llm_tool_calling"): + """Per provider: configured, declared-capable, in cooldown, and WHY — contract C5's payload. + + ⭐ ONE LIST, ONE DOOR. The Assistant's model picker and the Agent chat's toggle (W36-T34) read + THIS, so a model offered in one place cannot be missing from the other, and neither can offer a + provider the ladder would refuse to call [[permitted-is-not-answerable]]. + """ + rows = [] + for name in LLM_DEFAULT_ORDER: + p = LLM_PROVIDERS[name] + cap = p.caps.get(capability) + rows.append({ + "provider": p.name, "label": p.label, "model": p.model, "wire": p.wire, + "configured": p.configured(), + "toolCalling": bool((p.caps.get("llm_tool_calling") or Capability(False)).capable), + "jsonMode": bool((p.caps.get("llm_json_mode") or Capability(False)).capable), + "capable": bool(cap and cap.capable), + "outOfCredit": no_credit(name), + "note": (cap.note if cap else ""), + }) + return rows + + +# ============================================================================================= +# THE CANONICAL SCHEMA — owner ruling 2026-08-08: +# *"standardize the schema between Bright Data and APIfy so we keep using the same pre-set +# database even if the underlying engine changes"* +# +# ⛔ THE PRESET TABLES ARE THE CONTRACT; A VENDOR IS AN IMPLEMENTATION DETAIL. `ut_ig_posts` and +# `ut_ig_post_snapshots` must not gain, lose or rename a column because a chain was reordered — a +# tenant's saved views, filters, rollups and forms all bind to these keys, and a schema that moves +# with the vendor turns a routing change into a data migration. +# +# So every provider normalises INTO the keys below and nothing reads a vendor row downstream. +# `verify_automation` asserts both normalisers emit exactly `CANONICAL_POST_KEYS` on a fixture, so +# a third provider cannot ship with a near-miss key like `viewCount` and silently write a column +# nobody declared. +# +# ⚠ ONE NAME PER MEASUREMENT, AND `views` IS THE MEASUREMENT INSTAGRAM DISPLAYS. Meta folded +# Impressions/Plays/Video Views into a single **Views** metric on 2025-04-10, so carrying both a +# `views` and a `plays` column would be modelling a distinction the platform deleted — and it is +# exactly the distinction that let an account-grain number wear the "Views" label for a month. +# ⛔ THE VENDOR KEY THAT LOOKS RIGHT IS THE WRONG ONE, ON BOTH VENDORS. Apify ships BOTH +# `videoViewCount` (10,678) and `videoPlayCount` (137,684) for one reel whose true displayed count +# is ~137K — and Bright Data's useless `views` is 10,638 for that same reel. The two vendors' junk +# fields AGREE with each other, which is precisely what makes picking by name so dangerous. +# canonical `views` <- apify `videoPlayCount` ✅ matches the grid +# canonical `views` <- apify `videoViewCount` ⛔ off by 13x +# canonical `views` <- brightdata `views` ⛔ off by 13x AND account-grain +# ============================================================================================= + +#: Every key a normalised POST row may carry. Absent > blank: a key is omitted when the provider +#: did not answer, because `upsert_rows` merges and an empty string would ERASE what an earlier +#: paid run learned. +CANONICAL_POST_KEYS = ( + "shortcode", "url", "influencer_key", "posted_at", "type", "caption", + "likes", "comments", "views", "paid_partnership", "partner", "hashtags", + "alt_text", "tagged_location", "source_payload", + # ⭐ 2026-08-09 — the three the SECOND provider answers and the first does not. Each has a + # matching `field_def` in the engine's POST_FIELDS; a key here without a column there is a + # value that normalises cleanly and is then dropped by the write door, silently. + "plays", "video_duration", "comments_disabled", +) + + +def _int_or_none(v): + """`-1` IS NOT A COUNT. Apify returns `likesCount: -1` when the creator HIDES their like + count — a real state that is not a measurement. Writing -1 would render as a negative like + count; writing 0 would claim nobody liked it. Both are lies, so the key is omitted.""" + try: + n = int(v) + except (TypeError, ValueError): + return None + return None if n < 0 else n + + +def normalize_post_apify(row): + """One Apify `instagram-scraper` item -> the canonical post row.""" + if not isinstance(row, dict): + return None + code = str(row.get("shortCode") or "").strip() + if not code: + return None + out = { + "shortcode": code, + "url": str(row.get("url") or f"https://www.instagram.com/reel/{code}/"), + "influencer_key": str(row.get("ownerUsername") or ""), + "posted_at": str(row.get("timestamp") or "").replace("T", " ")[:16], + "type": "video" if str(row.get("type") or "").lower() == "video" else + ("carousel" if row.get("childPosts") else "image"), + "caption": str(row.get("caption") or ""), + # ⭐ THE FIELD THIS WHOLE PROVIDER EXISTS FOR. `videoPlayCount`, never `videoViewCount`: + # the wrong one agreed with Bright Data's junk (10,678 vs 10,638) on a reel whose true + # count was ~137K, and `videoPlayCount` matched a browser read to the digit. + "views": _int_or_none(row.get("videoPlayCount")), + # ⭐⭐ 2026-08-09 (owner: *"Video plays (# Plays) field isn't in APIfy? i believe it is"*). + # They were right, and the `plays` COLUMN had been empty on all 816 rows because nothing + # ever wrote it — the field existed with no writer. MEASURED on two of their own reels: + # `videoPlayCount` 216,904 / 95,331 and `videoViewCount` 115,929 / 30,871. Two different + # real numbers, and we were storing only one of them. + # ⚠ `plays` TAKES THE PLAY COUNT, which is also what `views` carries today — so the two + # columns will agree until somebody decides otherwise, and that decision is the OWNER'S: + # re-sourcing `views` to `videoViewCount` would change what the ~480 rows already captured + # mean, and a column whose meaning changes halfway down is the one thing worse than a + # column with no data. Flagged rather than done. + "plays": _int_or_none(row.get("videoPlayCount")), + "likes": _int_or_none(row.get("likesCount")), + "comments": _int_or_none(row.get("commentsCount")), + # ⭐ FIELDS BRIGHT DATA DOES NOT RETURN AT ALL, kept because they are already paid for in + # this same response (owner: *"whatever APIfy has more than BD pls use it"*). + "video_duration": _int_or_none(row.get("videoDuration")), + "comments_disabled": "1" if row.get("isCommentsDisabled") else "", + "hashtags": ", ".join(str(h) for h in (row.get("hashtags") or []) if h), + "alt_text": str(row.get("alt") or ""), + "paid_partnership": "1" if row.get("paidPartnership") else "", + "partner": ", ".join(str((s or {}).get("username") or "") + for s in (row.get("sponsors") or []) if isinstance(s, dict)), + "tagged_location": str((row.get("locationName") or "")), + "source_payload": json.dumps(row, default=str)[:32_000_000], + } + return {k: v for k, v in out.items() if v not in (None, "")} + + +def normalize_profile_apify(row): + """One Apify `instagram-scraper` PROFILE item (`resultsType: "details"`) -> our snapshot shape. + + ⭐ WHY THIS EXISTS (owner report 2026-08-09: *"I need Bright Data and Apify to work correctly + in tandem"*). `PROVIDERS["apify"]` has declared `ig_profile` capable since 2026-08-08 and + `DEFAULT_CHAINS["ig_profile"]` has read `("brightdata", "apify")` — but **nothing ever ran + that chain.** `connectors_ig.pull_profile` went Bright Data -> anonymous HTML rungs and Apify + was never asked, so a profile Bright Data cannot scrape came back `blocked` while a + configured, declared-capable provider sat unused. A registry entry with no runner is a + promise the product does not keep ([[flag-shipped-without-its-writer]]). + + ⚠ THE KEYS ARE APIFY'S camelCase, and mapping them HERE is the boundary rule this module + already enforces for posts: nothing downstream may ever see a vendor-shaped key, so the + snapshot writer cannot tell which vendor answered — which is what makes the fallback + invisible to every consumer instead of a second schema. + + ⛔ OMIT, NEVER ZERO. A field Apify did not send is dropped, exactly as `normalize_post_apify` + drops a missing count: writing 0 followers would claim we measured an empty account, and the + caller's `satisfied` hook reads absence as "did not answer" and falls through. A zero would + stop the chain on a lie. + """ + if not isinstance(row, dict): + return None + # ⛔ AN ERROR ENVELOPE IS NOT A PROFILE (measured 2026-08-09). Apify answers a dead handle + # with a 200 and a well-formed item — `{"username": …, "error": "not_found", + # "errorDescription": "Post does not exist"}` — and this function used to build a "profile" + # out of it: a username, a url and a source_payload, with every measured field absent. It + # then read to the caller as a vendor that answered, so the reason was replaced by a shrug. + # Refused HERE as well as in `connectors_ig.apify_profile` on purpose: the boundary rule this + # module exists for is that nothing downstream ever sees a vendor-shaped key, and a vendor's + # ERROR shape is the one that must never become a row. + # ⚠ TRUTHINESS, NOT KEY PRESENCE — a successful item carries `error: null` (measured on + # `sriyynntt`), so `"error" in row` would refuse every good profile. + if row.get("error"): + return None + handle = str(row.get("username") or "").strip() + if not handle: + return None + out = { + "username": handle, + "full_name": str(row.get("fullName") or ""), + "bio": str(row.get("biography") or ""), + "followers": _int_or_none(row.get("followersCount")), + "following": _int_or_none(row.get("followsCount")), + "posts_count": _int_or_none(row.get("postsCount")), + "verified": "1" if row.get("verified") else "", + "external_url": str(row.get("externalUrl") or ""), + "ig_id": str(row.get("id") or ""), + "profile_url": str(row.get("url") or f"https://www.instagram.com/{handle}/"), + "business_category": str(row.get("businessCategoryName") or ""), + "is_business": "1" if row.get("isBusinessAccount") else "", + "is_private": "1" if row.get("private") else "", + "highlights_count": _int_or_none(row.get("highlightReelCount")), + "source_payload": json.dumps(row, default=str)[:32_000_000], + } + # ⚠ `followers`/`following` are ints and 0 is a LEGITIMATE value for them, so the filter + # below must not treat 0 as absent the way the post normaliser can — an account really can + # have zero followers. Only None and "" are dropped. + return {k: v for k, v in out.items() if v is not None and v != ""} + + +def canonical_gaps(row, want=("views",)): + """Which requested canonical fields this row does NOT carry — the `satisfied` input. + + Named rather than inlined because "did the vendor actually answer the question" is the whole + fallback trigger, and a chain that asks it differently in two places will drift. + """ + row = row if isinstance(row, dict) else {} + return [k for k in want if row.get(k) in (None, "")] diff --git a/api/routes_admin.py b/api/routes_admin.py index 287e66ef1c7bbffe10423d6497a47ee95749166c..2c049e137d9ea4f524a97b7b0c01c0225c5fd96e 100644 --- a/api/routes_admin.py +++ b/api/routes_admin.py @@ -1,1248 +1,1248 @@ -"""routes_admin.py — Y4: user administration + the session's own settings (W2-3). - -These routes are the standalone shell's replacement for `app.py`'s `users_dialog` / `settings_dialog` -modals, over the SAME `core/users` account store — so both front-ends administer one set of -accounts and the eventual OIDC migration (D-3) swaps the CREDENTIAL check, not the user model. - -ADMIN-ONLY, FAIL-CLOSED. `admin_gate` is a role check (`perms.is_admin`), mirroring the Streamlit -dialog's `if not is_admin()`. A default record has `role: 'user'`, so an unreadable or partial record -is denied rather than admitted. `verify_api.py` proves it by having a viewer TRY every route. - -⛔ THIS IS THE FIRST WRITER OF `modules` THAT HAS EVER EXISTED. `core.users.set_access(modules=…)` -has no caller anywhere in the shipped app — module grants are only settable by editing the store -JSON out of band. That matters because the record format carries TWO documented fail-OPENS, both -asserted in `verify_api.py` section A: - - * `modules: []` is FALSY, so `perms.allowed_modules` reads it as 'all' = UNRESTRICTED. An admin - clearing every checkbox to lock an account down would grant it everything. - * `bus: []` falls through `allowed_bus_labels`'s "no recognisable label" branch to - `['All','Fisch','Royal']` — so one typo'd BU id is FULL cross-BU access, in the model whose - whole point is strict isolation. - -Y4 says do not "fix" `modules: []` in this wave, and that is right: the READER is mirrored in -`ui/session.py` and `core/perms.py` and diverging one of them mid-wave breaks lock-step. But the -WRITER is new, and it can simply refuse to create either footgun. So both are 400s here and both -read semantics are untouched — the seam is write-strict / read-unchanged. - -⚠ A STORE OUTAGE IS A 503, NEVER AN EMPTY LIST. `users.registry()` swallows a failed read into `{}` -one level down, and serving that as `{"users": []}` would tell an administrator their tenant has no -accounts. Same rule as the write path: an empty 200 is never how this API says "something is wrong". -""" -import os -import re - -from fastapi import APIRouter, Body, Depends, Response - -import core.platform_admin as platform_admin # wave 19 R3 — the /settings chrome flag -import core.registry as registry -import core.store as store - -from deps import Session, err, perms, require_session, users -from routes_auth import _public_user - -router = APIRouter(prefix="/api/v1") - -#: A username is a STORE KEY (`users.json`) and also the key the table workspace is filed under -#: (`data[username]`), so it is constrained rather than trusted: lowercase, no separators, no -#: whitespace, nothing that could traverse or collide once it becomes part of a path or a filename. -_UNAME_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{1,31}$") -#: ⭐⭐ WAVE 37 · T03 (owner ruling R8) — AN EMAIL ADDRESS IS ALSO A USERNAME. Owner: the username -#: IS the email, one identity. ⛔ THIS IS NOT A NEW CAPABILITY, IT IS THE DOOR CATCHING UP WITH THE -#: DATA: measured on deployed staging before the change, FIVE of tenant #0's eight accounts already -#: carry an `@` in the `username` field (`david@fischfloralsupply.com`, `farhan@`, `florencia@`, -#: `karen@`, `naomi@`) — every one of them illegal under the slug pattern above and longer than its -#: 32-character cap, so this route could not have created any of them. -#: ⚠ DELIBERATELY PERMISSIVE, because the store key is the thing being constrained and not the -#: deliverability of the address: one `@`, a dot-bearing domain, and the same character class the -#: slug already trusts. A stricter grammar here would refuse addresses that exist. -_UNAME_EMAIL_RE = re.compile(r"^[a-z0-9][a-z0-9._%+-]*@[a-z0-9][a-z0-9.-]*\.[a-z]{2,24}$") -#: RFC 5321's cap on a whole address. The slug form keeps its own 32 via `_UNAME_RE`. -_UNAME_MAX = 254 -#: Passwords are PBKDF2-200k, which is only as strong as what it is given. The Streamlit dialog -#: enforces nothing; this does, and the two are allowed to differ because only one of them is -#: reachable from the internet. -_MIN_PW = 8 -_ROLES = ("user", "admin") - - -def admin_gate(session: Session = Depends(require_session)) -> Session: - """401 without a session, 403 without the admin role. Role, not a module grant — mirroring - `app.py`'s `users_dialog`, whose only wall is `is_admin()`.""" - if not perms.is_admin(session.user): - raise err(403, "forbidden", "administrators only") - return session - - -def _require_store(): - """A 503 the moment the store cannot serve, so no route below can report emptiness as truth.""" - try: - ok = store.available() - except Exception: - ok = False - if not ok: - raise err(503, "store_unavailable", - "the tenant store is unavailable — accounts cannot be read or changed") - - -def _registry(): - _require_store() - try: - reg = users.registry() or {} - except Exception: - raise err(503, "store_unavailable", "the tenant store is unavailable") - return reg - - -def _view(uname, rec, governed=None, st=None, surface_keys=None): - """What an administrator may see about an account. NEVER `salt` or `hash` — this function is - the only projection these routes use, so there is one place that can leak and it does not. - - ⚠ `governed`/`st` ride through to `_access_summary` and nowhere else. The ROSTER passes both, - resolved once for the whole request; the single-record routes pass neither, because their - caller fetches the editor payload separately and `GET /perms` is the authority there. - """ - return {"username": uname, - "name": rec.get("name") or uname, - "role": rec.get("role", "user"), - "bus": rec.get("bus", "all"), - "bu_labels": perms.allowed_bu_labels(rec), - "modules": rec.get("modules", "all"), - "agent": rec.get("agent") or None, - "email": rec.get("email") or None, - "tenant": _tenant_of(rec), - "active": bool(rec.get("active", True)), - # The session-revocation handle. An admin needs it: it is the only visible evidence - # that "sign them out everywhere" actually happened. - "epoch": int(rec.get("epoch") or 0), - # S3 ask 4 — one sentence for the roster's Access column, so the list does not need - # a /perms round trip per row to fill one cell. Computed from the same record the - # editor will open, so the two cannot disagree. - "accessSummary": _access_summary(rec, uname=uname, governed=governed, st=st, - surface_keys=surface_keys), - "perms_v": int(rec.get("perms_v") or 0)} - - -def _access_summary(rec, uname="", governed=None, st=None, surface_keys=None): - """One sentence describing what this account may reach. Deliberately says LESS than the - editor: it is a signpost, not a rule listing, and a summary that tried to spell out filters - would be wrong the moment a filter got interesting. - - ⛔⛔ W36-T22 — IT MUST COUNT WHAT THE EDITOR WOULD SHOW, NOT WHAT IS STORED, AND THE TWO STOPPED - AGREEING THE MOMENT `get_perms` LEARNED TO DEFAULT FROM `may_read`. This counted STORED ENTRIES - only, which WAS the same answer before this wave: a migrated account had an entry for every - governable database, because the editor wrote one per topic on every save and no `ut_*` entry - could exist at all. Now a `ut_*` key legitimately has NO entry and is still OPEN, so the roster - would say *"1 module"* about an account the editor shows reaching twelve. ⛔ The roster is the - screen an administrator reads FIRST, and two screens disagreeing about one account is the - defect this wave exists to remove, not a rounding difference. - - ⚠ `governed`/`st` ARE PASSED IN RATHER THAN DERIVED HERE, and that is a cost decision. This - runs once PER ROW of the roster; asking `may_read` per (account x database) without a lend - would be a whole-document copy per question — D-213's shape, one route over. The caller - resolves the tenant's list and ONE lend for the whole request. A caller that passes neither - keeps the stored-entry reading, which is right for the single-record routes: their client - fetches `GET /perms` separately and that payload is the authority. - """ - import core.perm_scope as perm_scope - - if perms.is_admin(rec): - # amendment 4: an admin bypasses `perms` entirely, so rendering their stored rules as if - # they applied is the one misreading of that clause that could actually hurt. - return "Everything (admin)" - if not perm_scope.is_migrated(rec): - mods = perms.allowed_modules(rec) - return "All modules (legacy rules)" if mods is None else \ - f"{len(mods)} module{'' if len(mods) == 1 else 's'} (legacy rules)" - entries = (rec.get("perms") or {}) - if governed: - principal = users._public(uname, rec) if uname else rec - # ⛔ PER KIND, MIRRORING `get_perms`' OWN DEFAULTS — the first draft asked - # `nav_may_open` for EVERY key and that was a second idea of the count: for a migrated - # record with no topic entry the editor's default is `may_read` (DENY), so a blanket - # admission read would make the roster say "3 modules" about an account whose editor - # shows one box ticked. The roster's whole contract (leg 6 of the T22 section) is that - # it counts what the editor would show; a SURFACE key defaults from admission there, - # everything else from `may_read`, so this does exactly the same, key by key. - opened = [k for k in governed - if (perm_scope.nav_may_open(principal, k, st=st) - if k in (surface_keys or ()) else - perm_scope.may_read(principal, k, st=st))] - else: - opened = [k for k, e in entries.items() if isinstance(e, dict) and e.get("access", True)] - restricted = sum(1 for k in opened - if isinstance(entries.get(k), dict) - and (entries[k].get("filter") or entries[k].get("hiddenFields"))) - if not opened: - return "No modules" - base = f"{len(opened)} module{'' if len(opened) == 1 else 's'}" - return f"{base}, {restricted} restricted" if restricted else base - - -# ── request validation: every rule below is fail-closed ────────────────────────────────────────── -def _clean_username(v): - """The ONE validator for a new account's key. R8 widened it to admit a full email address. - - ⛔ IT HAS EXACTLY ONE CALL SITE AND THAT IS LOAD-BEARING, not an oversight to tidy up. - `PATCH /admin/users/{username}`, the password route and both `/perms` routes take the username - as a PATH parameter and never re-validate it, which is precisely why the five email-keyed - accounts above are administrable today despite being unconstructable. Arming this validator on - those routes would look like hardening and would break five live accounts. - """ - uname = str(v or "").strip().lower() - if len(uname) > _UNAME_MAX or not (_UNAME_RE.match(uname) or _UNAME_EMAIL_RE.match(uname)): - raise err(400, "bad_username", - "a username is either 2 to 32 characters of lowercase letters, digits, dot, " - "dash or underscore, or a full email address of up to 254 characters") - return uname - - -def _clean_password(v): - pw = str(v or "") - if len(pw) < _MIN_PW: - raise err(400, "weak_password", f"a password must be at least {_MIN_PW} characters") - return pw - - -def _clean_role(v): - role = str(v or "").strip().lower() - if role not in _ROLES: - raise err(400, "bad_role", f"role must be one of {list(_ROLES)}") - return role - - -def _clean_bus(v): - """'all', or a non-empty list of KNOWN business-unit ids. - - Refusing an unknown id is the whole point: `allowed_bus_labels` treats a list with no - recognisable label as full access, so `bus: [7]` would be a cross-BU grant created by a typo. - """ - if isinstance(v, str): - if v.strip().lower() == "all": - return "all" - raise err(400, "bad_bus", "bus must be 'all' or a list of business-unit ids") - if not isinstance(v, (list, tuple)) or not v: - raise err(400, "bad_bus", - "bus must be 'all' or a NON-EMPTY list of business-unit ids (an empty list " - "would read as unrestricted)") - out = [] - for b in v: - try: - b = int(b) - except (TypeError, ValueError): - raise err(400, "bad_bus", "a business-unit id must be a number") - if b not in users.BU_LABELS: - raise err(400, "bad_bus", - f"{b} is not a known business unit {sorted(users.BU_LABELS)} — an " - "unrecognised id would widen access to every BU") - out.append(b) - return sorted(set(out)) - - -def _clean_modules(v): - """'all', or a non-empty list of KNOWN module keys. - - `[]` is refused because it READS as unrestricted (see the module docstring). An unknown key is - refused because `may_open` is fail-closed on it: an admin who typed `sale` for `sales` would - silently lock the account out of the page they meant to grant. - """ - if isinstance(v, str): - if v.strip().lower() == "all": - return "all" - raise err(400, "bad_modules", "modules must be 'all' or a list of module keys") - if not isinstance(v, (list, tuple)) or not v: - raise err(400, "bad_modules", - "modules must be 'all' or a NON-EMPTY list of module keys — an empty list " - "reads as UNRESTRICTED, which is the opposite of locking an account down") - known = set(registry.BY_KEY) | set(perms._LEGACY_KEYS) - out, bad = [], [] - for k in v: - k = str(k or "").strip() - (out if k in known else bad).append(k) - if bad: - raise err(400, "bad_modules", f"unknown module keys: {sorted(bad)}") - return sorted(set(out)) - - -def _tenant_of(rec): - """The account's company, with the pre-wave default. ONE spelling of the default, used by - every admin route — two spellings is how a tenant wall grows a gap.""" - return str((rec or {}).get("tenant") or "royal-imports").strip().lower() - - -# ── the routes ─────────────────────────────────────────────────────────────────────────────────── -@router.get("/admin/users") -def list_users(session: Session = Depends(admin_gate)): - """Wave 18 (C1-TENANT): scoped to the CALLER'S tenant. The user registry is a global - control-plane bucket, so without this filter a Nurilab admin would read Royal's whole - roster — names, emails, agents — from one URL.""" - reg = _registry() - mine = _tenant_of(session.user) - # ⭐ W36-T22 — the tenant's governable list and ONE lend, resolved once for the whole roster so - # `_access_summary` can agree with the editor without paying a document copy per question. - import core.user_tables as _ut - try: - _rows = _perm_modules(session, surfaces=True) - governed = [m["key"] for m in _rows] - surface_keys = {m["key"] for m in _rows if m.get("surface")} - lent = _ut.lend_defs(session.runtime) - except Exception: # noqa: BLE001 - # A store that cannot list the tenant's databases must not take the roster down. The - # summary degrades to the stored-entry reading, which is what it always was. - governed, lent, surface_keys = None, None, None - return {"users": [_view(u, reg[u], governed=governed, st=lent, surface_keys=surface_keys) - for u in sorted(reg) - if _tenant_of(reg[u]) == mine]} - - -@router.post("/admin/users", status_code=201) -def create_user(body: dict = Body(default=None), session: Session = Depends(admin_gate)): - """Create an account. **409 if the username already exists — `PATCH` is the update path.** - - ⛔ WHY 409 AND NOT AN UPSERT. `core.users.create_user` writes a fresh `_record()`, and a fresh - record has NO `epoch` — so overwriting an existing account used to drop its epoch back to 0 and - RESURRECT every cookie minted before its last password change. `core/users.py` now - preserves-and-bumps on overwrite (which also closes it for `app.py`'s dialog, where "Add / - update a user" calls the same function), but this route still refuses: an upsert that silently - replaces an account's password, role and BU access because someone reused a username is not an - update anybody asked for. - """ - body = body or {} - _require_store() - uname = _clean_username(body.get("username")) - pw = _clean_password(body.get("password")) - role = _clean_role(body.get("role") or "user") - bus = _clean_bus(body.get("bus") if body.get("bus") is not None else "all") - modules = _clean_modules(body.get("modules") if body.get("modules") is not None else "all") - name = str(body.get("name") or "").strip() or uname - agent = str(body.get("agent") or "").strip() or None - email = str(body.get("email") or "").strip() or None - - _reg_before = _registry() - if uname in _reg_before: - raise err(409, "user_exists", f"{uname} already exists — PATCH it to change it") - # ⭐⭐ WAVE 37 · T03 (R8) — THE COLLISION THE WIDENING CREATES, REFUSED AT CREATE TIME. - # `core/users.py::verify` resolves a typed string by looking up the registry KEY first and only - # scanning the `email` field when that misses (its own wave-18 R1 comment). That precedence was - # incidental while a username could not contain `@`; now it decides which of two accounts a - # person reaches, so it is stated in both files rather than left to be re-derived. - # ⛔ BOTH DIRECTIONS, because each one silently takes a login away from an account that has it: - # (a) a new USERNAME equal to somebody's EMAIL wins the key lookup and shadows their address; - # (b) a new EMAIL equal to somebody's USERNAME can never be reached, because their key wins. - # Refused with a named error rather than allowed and explained afterwards: the failure is a - # person signing in and landing in the wrong account, which nothing downstream can detect. - if "@" in uname: - _shadowed = next((k for k, r in _reg_before.items() - if isinstance(r, dict) - and str(r.get("email") or "").strip().lower() == uname), None) - if _shadowed: - raise err(409, "username_shadows_email", - f"{uname} is already the email address on the account {_shadowed}, which " - f"signs in with it today. This account would take that login over. Choose a " - f"different username, or clear the email address on {_shadowed} first.") - if email: - _e = email.strip().lower() - if _e != uname and _e in _reg_before: - raise err(409, "email_shadows_username", - f"{_e} is already a username on this deployment, so anyone signing in with " - f"it reaches that account and never this one. Choose a different email " - f"address for this account.") - # Wave 18 (C1-TENANT): an admin creates accounts in their OWN company, never another's — - # the tenant is stamped from the session, not taken from the body. - users.create_user(uname, pw, name, role=role, bus=bus, modules=modules, - agent=agent, email=email, tenant=_tenant_of(session.user)) - reg = _registry() - if uname not in reg: - # The store accepted the write and does not have it. Reporting 201 here would be the - # "200 over a write that evaporated" failure the whole store-outage rule exists to prevent. - raise err(503, "store_unavailable", "the account was not saved — try again") - return {"user": _view(uname, reg[uname])} - - -@router.patch("/admin/users/{username}") -def update_user(username: str, body: dict = Body(default=None), - session: Session = Depends(admin_gate)): - """Change access on an existing account. Only the keys PRESENT in the body change. - - ⚠ NO EPOCH BUMP, and that is not an omission. `deps._user_for` re-reads the record on every - request, so a narrowed role / BU / module grant applies to the target's LIVE session on its very - next call — bumping the epoch would only add a forced re-login on top. `active: false` is the - exception and it bumps, inside `core.users.set_active`, because a disabled account must lose its - session rather than keep working until the cookie expires. - """ - body = body if isinstance(body, dict) else {} - reg = _registry() - uname = str(username or "").strip().lower() - if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): - # Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one — - # 404, never 403, because "that account exists in another company" is itself a leak. - raise err(404, "no_such_user", f"no account named {uname!r}") - if not body: - raise err(400, "empty_patch", "no fields to update") - - unknown = sorted(set(body) - {"name", "role", "bus", "modules", "active", "agent", "email"}) - if unknown: - # A silently-ignored field is how a UI ends up believing it saved something it did not. - raise err(400, "unknown_fields", f"cannot update: {unknown}") - - is_self = uname == session.uname - role = _clean_role(body["role"]) if "role" in body else None - active = bool(body["active"]) if "active" in body else None - # THE ONE-CLICK LOCKOUT GUARD. APP_PASSWORD remains the real backstop for 'admin', so this is - # not the thing that keeps the product reachable — it just stops an administrator removing - # their own access with a toggle and having to go find the master password. - if is_self and role is not None and role != "admin": - raise err(400, "self_demote", - "you cannot remove your own administrator role — ask another admin") - if is_self and active is False: - raise err(400, "self_deactivate", "you cannot deactivate your own account") - - kw = {} - if role is not None: - kw["role"] = role - if "name" in body: - kw["name"] = str(body["name"] or "").strip() or uname - if "bus" in body: - kw["bus"] = _clean_bus(body["bus"]) - if "modules" in body: - kw["modules"] = _clean_modules(body["modules"]) - if "agent" in body: - kw["agent"] = str(body["agent"] or "").strip() # '' clears the link - if "email" in body: - kw["email"] = str(body["email"] or "").strip() - if kw: - users.set_access(uname, **kw) - if active is not None and active != bool(reg[uname].get("active", True)): - users.set_active(uname, active) # bumps the epoch: kills live sessions - - fresh = _registry() - return {"user": _view(uname, fresh[uname])} - - -@router.post("/admin/users/{username}/password", status_code=204) -def set_password(username: str, body: dict = Body(default=None), - session: Session = Depends(admin_gate)): - """Rotate a password. **Revokes every outstanding session for that account** — the epoch bump - happens inside the same read-modify-write as the hash (`core.users.set_password`), so the two - can never disagree. - - 204 with no body: there is nothing to say that the caller did not already know, and echoing the - account back would invite a client to diff a response for a password change. - """ - reg = _registry() - uname = str(username or "").strip().lower() - if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): - # Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one — - # 404, never 403, because "that account exists in another company" is itself a leak. - raise err(404, "no_such_user", f"no account named {uname!r}") - pw = _clean_password((body or {}).get("password")) - before = int(reg[uname].get("epoch") or 0) - users.set_password(uname, pw) - after = int((_registry().get(uname) or {}).get("epoch") or 0) - if after <= before: - # The whole value of this route is the revocation. If the epoch did not move, the sessions - # were not revoked, and answering 204 would say they were. - raise err(503, "store_unavailable", - "the password change did not persist — outstanding sessions were NOT revoked") - return Response(status_code=204) - - -# ── C-PERM (wave 15): per-module access + permanent filters + hidden fields ────────────────── -#: The modules the permission editor governs — the GRID surfaces, the ones with a field schema -#: to filter and hide. Amendment 6 shipped the wall on `customer_data` alone and said this list -#: is the one place that changes when C-TOPIC lands. WAVE 16: it landed, so `product_data` -#: joins — and it is not cosmetic. `perms_v: 1` means an UNDECLARED module DENIES (amendment 4), -#: so a migrated account would be locked out of Product with no control anywhere to grant it: -#: fail-closed, but unadministrable. The editor grows its section with zero client edits -#: (S3 built it off this route's `modules` + `fields_by_module`). -#: -#: ⭐⭐ WAVE 33 (owner item 11, W33-T33) — **THIS IS A CATALOGUE, NOT AN ANSWER.** It used to be -#: both, and that was the defect the owner flagged four times: `get_perms` served this tuple -#: verbatim and never touched `session.runtime`, so EVERY tenant was handed tenant #0's two grid -#: modules. What this names now is a fact about THIS FILE — the registry topics `_module_fields` -#: below has a field-schema arm for — and the answer a tenant receives is `_perm_modules(session)`, -#: which filters this by the tenant's own provisioning and merges the tenant's own `ut_*` -#: databases. ⛔ Serving this list directly again re-opens item 11; `verify_perm_scope`'s -#: `section_tenant_derived` NC does exactly that and reds. -#: ⭐ CONTRACT C3, CONSUMED (lane E's `W33-T46`, answered as `E-3`). The topic list is DERIVED from -#: `registry.governable_modules()` — `{topic: {'key','label','subject'}}` over every non-archived, -#: non-group_only registry row that declares a `topic`. What stays hand-written is -#: `_FIELD_PROVIDER_KEYS` below, and that is CODE, not policy: each name has its own `aios_grid` -#: contract behind it in `_module_fields`, and a topic with no arm cannot be governed by this -#: editor at all. The intersection is the honest answer to "what can this module build pickers for". -#: -#: ⚠ KEPT AS A SYMBOL RATHER THAN DELETED, deliberately. `verify_api`'s W30a reads -#: `routes_admin._PERM_MODULES` to assert `perm_scope.BU_FILTERABLE_MODULES` names exactly the -#: governed topics whose contract carries `dba`. Deleting the name would make that gate raise -#: `AttributeError` — a CRASH, which scores as "the gate is broken" rather than as a red -#: [[gate-must-go-red-not-crash]] — and `verify_api.py` is lane A's fence. Derived-and-kept gives -#: that assertion a truer subject than the literal ever did, and costs nobody a cross-fence edit. -_FIELD_PROVIDER_KEYS = frozenset({"customer_data", "product_data"}) - - -def _governable_catalogue(): - """The REGISTRY topics this module can govern: `[{key, label}]`, in registry order. - - ⛔ No tenant filtering here — that is `perms.tenant_governable_modules`' job, and E's map is - tenant-blind by design. Two filters in two files answering one question is how the perms route - and `routes_nav` came apart in the first place. - """ - import core.registry as registry - return [{"key": v["key"], "label": v["label"]} - for v in registry.governable_modules().values() - if v.get("key") in _FIELD_PROVIDER_KEYS] - - -_PERM_MODULES = tuple(m["key"] for m in _governable_catalogue()) - - -def _perm_modules(session, surfaces=False): - """The databases THIS tenant's permission editor governs: `[{key, label, enforced}]`. - - Three inputs, none of them a literal: the topic catalogue above, the tenant's provisioned - module set (`tenant.config['modules']`, exactly the rule `routes_nav.py::nav` applies), and - the tenant's own `ut_*` databases read through the DEFINITIONS projection. - - ⭐ `user_tables.nav_entries` rather than a fresh listing, deliberately: it is already the ONE - resolver for "which databases may this account see", it is `may_open`-filtered, and it reads - the projected document (`all_defs`) so this costs definitions, never the 28.6 MB of rows. - The caller is admin-gated, so the filter admits the whole tenant — but asking the resolver - instead of assuming that is what keeps this from becoming the SECOND idea of who may see a - table (`user_tables.may_open`'s own docstring is the record of the first time that happened). - """ - import core.perms as _perms - import core.user_tables as user_tables - try: - ut = user_tables.nav_entries(viewer=session.uname, is_admin=True, - st=session.runtime) - except Exception: - # A store that cannot list the tenant's databases must not take the whole editor down — - # the topic half is still administrable, and an empty `ut_*` half is visibly empty. - ut = () - cat = _governable_catalogue() - skeys = set() - if surfaces: - # ⭐⭐ OWNER RULING 2026-08-18 — the Assistant and Agents surfaces are - # governable ACCESS toggles in the ACCOUNT editor. `surfaces` is caller-explicit and - # defaults CLOSED because the other consumer of this list is the CHANNEL-AGENT editor - # (`routes_slack.get_channel_agent`), whose principal is enforced per DATABASE - # (`agent_rows`/`agent_may_read`) and never opens an app surface: an Assistant toggle - # there would be a stored rule nothing applies, the exact class C2 deleted. - surf = _perms.governable_surface_modules() - skeys = {s["key"] for s in surf} - cat = cat + surf - rows = _perms.tenant_governable_modules(session.runtime, cat, ut_entries=ut) - # The marker survives OUTSIDE `tenant_governable_modules`, which builds bare {key,label} - # rows: the client needs it only to choose the schemaless help copy, and the server needs - # it only to pick the admission default in `get_perms`. - return [dict(r, surface=True) if r["key"] in skeys else r for r in rows] - - -#: ⛔ `_enforced_keys` IS DELETED (W36-T22 / CONTRACT C2). It asked `perms.enforced_module_keys` -#: which subset of the listed databases a wall could actually be stored against, and after R6 the -#: answer is "all of them" — a question with one possible answer is not a question. Both it and the -#: `perms` helper behind it are gone rather than made to return everything, because a predicate -#: that ignores its argument is the always-true flag C2 deletes, one indirection along. -#: What callers use instead is `{m["key"] for m in _perm_modules(session)}` — the SAME list the -#: editor renders, so "shown" and "storable" cannot drift apart again. - - -def _module_fields(key, session=None): - """The field schema the editor builds its pickers from — the SAME vocabulary the grid uses, - so an admin filters on exactly what the user sees. - - ⛔ Returned WITHOUT any per-user hiding applied, deliberately: this is the admin choosing - what to hide, so hiding the choices would make a field unhideable the moment it was hidden - once for somebody. The route is admin-gated; the projection a MEMBER receives is the one - `perm_scope.visible_fields` narrows. - - ⚠ PER TOPIC (wave 16). `product_data` has its OWN canonical contract, and handing the - editor the customer schema for it would let an admin build a wall out of columns that - module does not have — `_clean_perms` would then 400 on the admin's own choices, or worse, - a filter naming a field the product rows lack would DENY every row via `permits()`. - The product contract is served CONSOLIDATED (every column) on purpose: a BU-scoped reader - is served fewer columns, but the ADMIN is choosing what may ever be hidden. - - ⛔⛔ WAVE 33 — THE `else` USED TO BE A FALL-THROUGH, AND IT WAS SILENT-WRONG. The body read - `aios_grid.product_fields() if key == "product_data" else aios_grid.FIELDS`, so ANY key that - was not `product_data` got the CUSTOMER contract: the moment the governed list stopped being - a two-element literal, a `ut_*` database or a third topic would have been handed customer - columns, `_clean_perms` would have validated an admin's `hiddenFields` against them, and the - stored wall would name columns that module does not have — which `permits()` resolves by - DENYING every row. No error at any point. So the map is explicit and an unknown key RAISES: - a topic without a schema arm is a missing arm, never a customer grid in disguise. - - ⭐⭐ W36-T22 / R6 — AND `ut_*` NOW HAS AN ARM, WHICH IS THE HALF THAT MAKES THE TOGGLE REAL. - Owner item 11: *"how come the database is only toggleable for Odoo customers and Odoo - products … EVERY database should be able to be toggleable by admin."* A `ut_*` database's - field contract is its OWN stored definition, so the picker offers exactly the columns that - database has. ⛔ ONE SOURCE, NOT TWO: the picker (`get_perms.fields_by_module`) and the - validator (`_clean_perms`) both call this, so a column the picker offers can never be a - column the validator rejects. Getting that wrong is worse than an error message — - `permits()` DENIES on a predicate it cannot answer, so a mismatched wall is a - deny-everything trap wearing a saved-successfully toast. - - ⚠ `session` IS REQUIRED FOR A `ut_*` KEY and unused for a topic: a stored definition is - per-tenant data and there is no tenant-blind way to read one. A caller that omits it is - refused LOUDLY rather than handed another tenant's schema — the same rule the `else` - fall-through above was deleted for. - """ - import aios_grid - key = str(key or "") - if key in {s["key"] for s in perms.governable_surface_modules()}: - # ⭐ A SURFACE (Assistant, Agents) has NO field schema, and `[]` is the honest - # answer rather than the wave-26 empty-option trap: the editor's schemaless rendering - # is a deliberate state (access toggle only, R9), and `_clean_perms` refuses any - # filter or hiddenFields validated against this empty vocabulary. An unknown key - # still RAISES below — this arm admits exactly the derived surface list. - return [] - if key.startswith("ut_"): - if session is None: - raise err(400, "ungoverned_module", - f"{key!r} is a tenant database, so its columns can only be read for a " - f"known tenant — this caller passed no session") - import core.user_tables as user_tables - defn = user_tables.get(key, st=user_tables.lend_defs(session.runtime)) - if not defn: - raise err(400, "ungoverned_module", - f"{key!r} is not a database in this workspace") - src = defn.get("fields") or [] - else: - providers = {"customer_data": lambda: aios_grid.FIELDS, - "product_data": aios_grid.product_fields} - provider = providers.get(key) - if provider is None: - # Reached only by a caller that skipped `_perm_modules`. Loud, because the alternative - # (`[]`) is an empty option list read as an ANSWER — wave 26 item 24, `if ([])` is - # truthy. - raise err(400, "ungoverned_module", - f"{key!r} has no permission-editor field schema. The editor governs " - f"{sorted(providers)} and this tenant's own databases") - src = provider() - known = _server_side_vocabularies() - out = [] - for f in src: - if not isinstance(f, dict) or not f.get("key"): - continue - opts = f.get("options") - if not isinstance(opts, list) or not opts: - # ⭐ D-76 / W33-T35 — SUPPLY WHAT THE SERVER ALREADY KNOWS. See - # `_server_side_vocabularies`. - opts = known.get(f["key"]) - out.append({"key": f["key"], "label": f.get("label") or f["key"], - "type": f.get("type") or "text", - **({"options": list(opts)} if isinstance(opts, list) and opts else {}), - **({"pinned": True} if f.get("pinned") else {})}) - return out - - -def _server_side_vocabularies(): - """`{field key: [choice, …]}` for choice columns whose vocabulary is CLOSED and known here, - but which the grid contract does not declare. - - ⛔ D-76 — WHY THE PERMISSION EDITOR'S DROPDOWNS WERE EMPTY, AND WHY THE FIX IS HERE. The grid - resolves a choice column through `types.ts::choiceVocabulary`: the DECLARED list when the - field has one, otherwise the values seen in the loaded ROWS. This editor loads no rows, so - every column that leans on the second branch renders `Select…` with nothing in it — the - identical shape wave 26 item 24 closed on the Product grid, one surface over. ⚠ And the fix - belongs at the CALLER, never in the panel: `FilterBuilderPanel::choicesFor` treats a supplied - list as authoritative *including when it is empty* ("this column has no values" is an answer), - which is correct and must not be weakened. So the caller stops handing it nothing. - - ⭐ IMPORTED, NEVER RESTATED. `stock_bucket`'s labels are `modules/inventory.COVERAGE_LABELS`, - the same constant `_bucket()` mints from — a copy here would be a second vocabulary that - drifts the first time a band is renamed [[constant-two-features-share]]. The import is lazy - for the same reason `aios_grid`'s is: this route should not pull the analytics stack at - module load. - - ⚠ TWO COLUMNS ARE DELIBERATELY ABSENT, and R6's second sentence says to name them rather than - let them look handled: - · `category` (product) — an OPEN vocabulary read off Odoo's product categories. There is no - closed list to declare, and discovering it would mean a pool read on an admin route. - · `odoo_status` (customer) — WAS bare for the same reason and is NOT any more: lane E - answered `ASK D-15` by declaring `options: ['Active','Archived']` on the CONTRACT, which - is the right home (the grid picks a declared list up for free) and is why this function - never needed an arm for it. Recorded because the ask, not a guess, is what settled it. - """ - try: - import modules.inventory as inventory - labels = list(inventory.COVERAGE_LABELS or ()) - except Exception: - # A missing analytics import must not take the permission editor down; the dropdown - # degrades to today's empty one rather than the page to a 500. - return {} - return {"stock_bucket": labels} if labels else {} - - -def _clean_perms(v, governed_keys=None, session=None): - """Validate a whole `perms` block. FAIL-CLOSED, and LOUD rather than lenient. - - ⭐ WAVE 33 (W33-T33) — `governed_keys` IS PER TENANT AND IT IS NOT OPTIONAL IN PRACTICE. It - used to be `set(_PERM_MODULES)`, a literal, which is the same tenant-blindness owner item 11 - flagged: an admin of a tenant provisioned for neither grid module could still store a wall - for both. It now comes from `_perm_modules(session)`, i.e. from the tenant's own catalogue. - The default is the topic catalogue itself so a test or a future caller cannot silently widen - it. - - ⭐⭐ W36-T22 / CONTRACT C2 — **THE `enforced`/`listed` SPLIT IS GONE, AND SO IS - `unenforced_module`.** This function took TWO key sets and refused anything in the gap - between them, with a message that ended *"Share the database instead, or arm `perm_scope` - over `ut_*` first."* W36-T21 armed it. There is no gap left: every database the editor lists - is a database whose wall `routes_tables` applies on every read, so a wall stored against one - is enforced by the same code that enforces `customer_data`'s. **Two sets collapse into one**, - which is the shape C2 asks for — not a second set that is always equal to the first. - - ⚠ `session` IS THREADED SO `_module_fields` CAN READ A `ut_*` SCHEMA. Without it the - validator would have no field contract for the databases this ticket just made governable, - and "no contract" resolves to "every `hiddenFields` entry is unknown" — a 400 on the admin's - own screen. - - ⛔ WHY THIS REFUSES WHERE `clean_filter_tree` DROPS. The view sanitiser is deliberately - drop-per-node: one bad rule must never cost a user their whole saved view. A PERMISSION - filter is the opposite situation — an admin types "agent is Tara", a leaf is silently - dropped, and the stored wall is EMPTY. The admin sees "Saved", the account sees the whole - book, and nothing anywhere says so. So anything that would be dropped is a 400 instead: - the filter is re-validated with `clean_filter_tree` and the result must come back with the - SAME leaf count it went in with. - - Shape (C-PERM amendment 2 — a `FilterTree` is a PAIR, `{conj?, nodes}`; `clean_filter_tree` - validates the NODE LIST alone and never sees the root conjunction, so the two are checked - separately and a bare node list is refused rather than silently read as `and`). - """ - import aios_grid - - if v is None: - return None - if not isinstance(v, dict): - raise err(400, "bad_perms", "perms must be an object keyed by module") - governed = set(governed_keys) if governed_keys is not None else set(_PERM_MODULES) - unknown = sorted(set(v) - governed) - if unknown: - raise err(400, "bad_perms", - f"unknown module keys: {unknown} — this tenant's permission editor governs " - f"{sorted(governed)}") - out = {} - for key, raw in v.items(): - if not isinstance(raw, dict): - raise err(400, "bad_perms", f"{key}: each entry must be an object") - valid_keys = {f["key"] for f in _module_fields(key, session=session)} - hidden = raw.get("hiddenFields") or [] - if not isinstance(hidden, list): - raise err(400, "bad_perms", f"{key}: hiddenFields must be a list") - bad = sorted({str(h) for h in hidden} - valid_keys) - if bad: - # A hiddenFields entry naming nothing hides nothing — and reads as a restriction - # that is not there. - raise err(400, "bad_perms", - f"{key}: hiddenFields names unknown fields {bad}") - tree = raw.get("filter") - clean_tree = None - if tree not in (None, {}, []): - if not isinstance(tree, dict) or not isinstance(tree.get("nodes"), list): - raise err(400, "bad_filter", - f"{key}: filter must be {{conj?, nodes:[…]}} — a bare list would lose " - f"the root conjunction, and an 'or' wall read as 'and' restricts " - f"nothing it was meant to") - conj = tree.get("conj", "and") - if conj not in ("and", "or"): - raise err(400, "bad_filter", f"{key}: conj must be 'and' or 'or'") - nodes = tree["nodes"] - cleaned = aios_grid.clean_filter_tree(nodes, valid_keys, cohort_ids=None) - if _leaf_count(cleaned) != _leaf_count(nodes): - raise err(400, "bad_filter", - f"{key}: the filter contains conditions this module cannot evaluate " - f"(an unknown field, an unknown operator, or a cohort leaf — cohorts " - f"are per-user and cannot be a permanent rule). Refused rather than " - f"saved with the bad conditions silently removed, which would store a " - f"weaker wall than the one on screen.") - clean_tree = {"conj": conj, "nodes": cleaned} - # ⭐⭐ W38-T19 — `metrics` IS THE FOURTH FIELD OF AN ENTRY, AND ITS DEFAULT IS GRANT. - # `raw.get("metrics", True)` rather than a required key: a PUT composed by an older - # client, or a copy of a record written before this ticket, must not read as a - # revocation. `perm_scope.may_metrics` applies the identical rule on the read side, so - # the validator and the wall cannot disagree about what an absent key means. - out[key] = {"access": bool(raw.get("access", True)), - "filter": clean_tree, - "hiddenFields": sorted({str(h) for h in hidden}), - "metrics": bool(raw.get("metrics", True))} - _refuse_unshapeable_bu(out) - return out - - -def _refuse_unshapeable_bu(perms_out): - """⛔ A BU CONDITION THE PUSHDOWN CANNOT READ IS A VALUE LEAK WEARING A CORRECT ROW LIST. - - Amendment 3 in one more place. `perm_scope.derive_pool_scope` recognises exactly the shapes - that PIN a business unit: a top-level `dba eq` leaf, or a top-level OR-group of them — which - is what `perm_migrate` emits and what the condition builder produces for "is any of". An - admin composing the same intent a slightly different way (a NESTED group, `dba neq Royal`, a - `dba` leaf one level down) produces a filter that still narrows the ROWS correctly through - `permits()` — and leaves the pool built CONSOLIDATED, so every surviving row carries - Fisch+Royal numbers. The row list looks right. The revenue is another BU's. - - That is precisely the defect amendment 3 exists for, re-entering through the editor R9 just - shipped, and it cannot be caught downstream: by then the numbers are simply wrong, with - nothing anomalous about them. - - So it is refused at the WRITE, where a person is present to fix it. The discriminator is - narrow on purpose — the filter must MENTION `dba` and must fail to pin a team. A wall like - `dba is Fisch OR revenue > 1000` genuinely does not confine anyone to one BU, and - consolidated values are the correct answer for it, so it is not caught here (`derive` also - refuses it) and must not be. - """ - import core.perm_scope as perm_scope - - for key, e in perms_out.items(): - tree = e.get("filter") - if not tree or not _mentions_dba(tree.get("nodes")): - continue - probe = {"role": "user", "perms_v": perm_scope.PERMS_VERSION, - "perms": {key: {"access": True, "filter": tree, "hiddenFields": []}}} - team_id, _ = perm_scope.derive_pool_scope(probe, key) - if team_id is None and _confines_to_one_bu(tree): - raise err(400, "bu_condition_shape", - f"{key}: this filter restricts the business unit in a way the data layer " - f"cannot push into the query, so the rows would be correct while the " - f"revenue figures on them stayed consolidated across both units. Express " - f"the business-unit condition as a TOP-LEVEL condition — 'DBA is Fisch', " - f"or an 'any of' over the brands — and keep any other rules alongside it.") - - -def _guard_bu_widening(rec, cleaned, confirmed): - """⛔ THE TWO-CLICK LEAK: a SECOND save that silently drops the business-unit condition. - - `_fold_legacy_scope` runs only while a record is un-migrated, which is correct — after the - first save the filter IS the whole wall and the admin owns it. But the first save also - CHANGES the tree (it folds the BU condition in), so a client holding the tree it submitted - rather than the one the server returned will, on its next save, PUT a payload with no BU - condition — and a whole-record replace deletes it. Measured, not theorised: the same client - payload saved twice takes the pool scope from `(6, 'Ann')` to `(None, 'Ann')`, which is both - business units. - - So a save that REMOVES the brand confinement is refused unless it says it means to. Widening - BU access is a legitimate thing for an administrator to do — it is just never a thing to do - by accident, and the difference between the two is one explicit flag. - - ⚠ Deliberately NOT solved by always folding the legacy scope: that would make a BU - restriction permanent and unremovable, which contradicts R1 (the filter is the wall, and the - admin edits it). The right shape is "you may widen, say so". - """ - import core.perm_scope as perm_scope - - if confirmed: - return - stored = (rec.get("perms") or {}) if isinstance(rec.get("perms"), dict) else {} - for key, new_entry in cleaned.items(): - if not new_entry.get("access", True): - # Revoking the module entirely is the OPPOSITE of widening, and the filter on a - # denied entry governs nothing. Guarding it would make "lock this account out" - # require a confirm_widen flag, which reads as nonsense to whoever hits it. - continue - old = stored.get(key) - if not isinstance(old, dict): - continue # nothing stored yet: the fold already handled it - old_tree, new_tree = old.get("filter"), new_entry.get("filter") - if not old_tree or not _confines_to_one_bu(old_tree): - continue # was not confined -> this save cannot widen it - if new_tree and _confines_to_one_bu(new_tree): - continue # still confined -> not a widening - raise err(400, "bu_widening", - f"{key}: this would REMOVE the business-unit restriction and give the account " - f"both units. If that is intended, resend with confirm_widen: true. If it is " - f"not, reload the account's permissions first — saving a filter loaded before " - f"the last change drops the conditions it did not know about.") - # A guard is only as good as its trigger: `perm_scope` is imported so a future reader sees - # the connection to derive_pool_scope, which is what the removed condition was feeding. - _ = perm_scope - - -def _mentions_dba(nodes): - for n in nodes or (): - if not isinstance(n, dict): - continue - if isinstance(n.get("children"), list): - if _mentions_dba(n["children"]): - return True - elif n.get("colId") == "dba": - return True - return False - - -def _confines_to_one_bu(tree): - """Does the BRAND STRUCTURE ALONE confine the reader to one business unit? - - ⚠ "Alone" is the whole precision of this function, and getting it wrong makes the guard - refuse legitimate walls. `dba is Fisch OR ar_open > 1000` does NOT confine anybody — a - high-balance Royal customer satisfies it — so consolidated values are the correct answer and - refusing it would be a false positive. My first attempt evaluated synthetic rows carrying - only `dba`, which made `ar_open > 1000` read false on every probe row and turned that exact - wall into a "confinement" it is not. - - So every NON-`dba` leaf is treated as TRUE — the most generous reading, i.e. "suppose the - reader satisfies everything else; can they still see both brands?" If yes, the wall does not - confine and the pool is correctly consolidated. If no, the brand structure is doing the - confining and `derive_pool_scope` must be able to see it, or the numbers are wrong. - """ - def admits(node, brand): - if isinstance(node.get("children"), list): - kids = [admits(k, brand) for k in node["children"] if isinstance(k, dict)] - if not kids: - return True - return any(kids) if node.get("conj") == "or" else all(kids) - if node.get("colId") != "dba": - return True # every other condition: assume satisfied - want = str(node.get("value") or "").strip().lower() - op = node.get("op") - if op == "eq": - return brand.lower() == want - if op == "neq": - return brand.lower() != want - if op == "contains": - return want in brand.lower() - if op == "doesNotContain": - return want not in brand.lower() - if op == "isEmpty": - return False # a probe brand is never blank - if op == "isNotEmpty": - return True - return True # an op that cannot judge a brand does not confine - - nodes, conj = (tree.get("nodes") or []), tree.get("conj", "and") - root = {"conj": conj, "children": nodes} - admitted = {b for b in ("Fisch", "Royal", "Both") if admits(root, b)} - if not admitted: - return False # admits nothing at all — a different problem - # 'Both' belongs to either unit, so it cannot by itself widen the answer. - return not ({"Fisch", "Royal"} <= admitted) - - -def _leaf_count(nodes): - n = 0 - for node in nodes or (): - if isinstance(node, dict) and isinstance(node.get("children"), list): - n += _leaf_count(node["children"]) - else: - n += 1 - return n - - -def _fold_legacy_scope(rec, cleaned): - """⛔ MIGRATING A RECORD MUST NEVER WIDEN IT. The invariant this function exists to hold. - - Writing perms stamps `perms_v: 1`, and that marker SWITCHES OFF the legacy `bus`/`agent` - fallback in `perm_scope` (amendment 4 — absence must start meaning DENY). So a first write - whose filter carries no BU condition silently promotes a Royal-only account to BOTH business - units: the wall that used to come from `bus` is gone and nothing replaced it. - - Caught by the end-to-end leg in `verify_api` and by nothing else — every model-level check - passed, because the model was doing exactly what it was told. The account simply received - the other BU's customers on the wire. - - So on the FIRST perms write we fold the legacy scope in, as ordinary conditions, exactly as - `perm_migrate` would: this IS R1's migration, performed at the moment the record acquires a - perms block. From then on the admin sees those leaves in the editor (GET returns the folded - filter) and can change them like any other condition — which is R1's whole point, and why - this is a one-time fold rather than a permanent floor AND-ed on every write. - - Idempotent by construction: it runs only while the record is un-migrated, and the write that - calls it is the write that migrates it. - - ⛔ THE FOLD IS PER MODULE, BECAUSE THE LEGACY FILTER IS CUSTOMER-SHAPED AND THE MODULES ARE - NOT. `perm_migrate.filter_for` speaks `dba` and `agent` — both CUSTOMER columns. `product_data` - has neither (19 fields, no brand column at all), and `apply_row_scope` evaluates the permanent - filter with `permits()`, which DENIES anything unanswerable. So folding the customer wall into - the product entry does not narrow the product grid, it EMPTIES it: measured at 0 rows of 2 for - a `bus:[5]` account the moment an admin pressed Save. - - Nothing was wrong upstream — `_clean_perms` already refuses a `dba` condition typed against - `product_data` (its leaf keys are validated per topic). This fold was the ONLY door a - cross-topic leaf could come through, which is why it is the only place that needs the rule. - - ⚠ WHAT IS LOST HERE IS RECOVERED IN `perm_scope.derive_pool_scope`, NOT DISCARDED. Dropping the - BU condition from an AND root WIDENS that module's row scope, and for the product grid the BU - is a VALUES question anyway (`team_id` shapes `rev_ytd`/`qty_ytd`/`inv_value` — amendment 3). - The pushdown falls back to the record's own `bus` for exactly the modules whose field - vocabulary cannot express a BU, so the product pool is still BUILT as Fisch. The two halves - ship together or the second one is a leak. - """ - import core.perm_migrate as perm_migrate - import core.perm_scope as perm_scope - - if perm_scope.is_migrated(rec): - return cleaned - legacy = perm_migrate.filter_for(rec) - if not legacy: - return cleaned # nothing to preserve: the account was unrestricted - out = {} - for key, e in cleaned.items(): - legacy_here = _prune_to_module(legacy, {f["key"] for f in _module_fields(key)}) - tree = e.get("filter") - if not legacy_here: - # This module cannot evaluate ANY of the legacy conditions. Fold nothing rather than - # storing a wall it will read as "deny every row". - out[key] = e - continue - if not tree or not tree.get("nodes"): - folded = legacy_here - else: - # AND the two roots together. The submitted tree keeps its own conjunction by being - # nested as a GROUP — flattening an `or` tree into an `and` root would turn the - # admin's "A or B" into "A and B", which is a different and much narrower wall. - folded = {"conj": "and", - "nodes": list(legacy_here["nodes"]) + [{"conj": tree.get("conj", "and"), - "children": list(tree["nodes"])}]} - out[key] = dict(e, filter=folded) - return out - - -def _prune_to_module(tree, valid_keys): - """`tree` with every leaf this module cannot evaluate removed, or None if nothing survives. - - Used ONLY on the legacy fold above, where the alternative to pruning is a filter that denies - every row. It is not a general sanitiser: `_clean_perms` REFUSES rather than drops, for the - reason its own docstring gives (a silently-weakened wall reads as "Saved"). - - ⚠ A GROUP THAT LOSES ANY CHILD IS DROPPED WHOLE. Half of an `or` group is a narrower rule than - the admin's intent and half of an `and` group is a wider one; neither is the thing that was - written, and a fold has no standing to invent a third meaning. All-or-nothing per group keeps - the surviving conditions ones somebody actually declared. - """ - if not isinstance(tree, dict): - return None - - def keep(node): - if not isinstance(node, dict): - return None - kids = node.get("children") - if isinstance(kids, list): - surviving = [keep(k) for k in kids] - if not kids or any(s is None for s in surviving): - return None - return dict(node, children=surviving) - return node if node.get("colId") in valid_keys else None - - nodes = [n for n in (keep(n) for n in tree.get("nodes") or ()) if n is not None] - return {"conj": tree.get("conj", "and"), "nodes": nodes} if nodes else None - - -@router.get("/admin/users/{username}/perms") -def get_perms(username: str, session: Session = Depends(admin_gate)): - """This account's permission block plus the field schema the editor needs to render it.""" - reg = _registry() - uname = str(username or "").strip().lower() - if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): - # Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one — - # 404, never 403, because "that account exists in another company" is itself a leak. - raise err(404, "no_such_user", f"no account named {uname!r}") - import core.perm_scope as perm_scope - - rec = reg[uname] - stored = rec.get("perms") or {} - # S3 ask 3 — AN ENTRY FOR EVERY DECLARED MODULE, never a gap the client has to default. - # The PUT is a whole-record replace over this same module list, so a key declared here and - # omitted from `perms` would become an explicit DENY the moment anyone pressed Save. The - # default sent is the EFFECTIVE one (`may_access`), so an un-migrated record shows the - # access its legacy grant currently gives rather than a guess in either direction. - # - # ⭐⭐ WAVE 33 (W33-T33, owner item 11) — THE LIST IS THIS TENANT'S, NOT A LITERAL. Everything - # below was built from `_PERM_MODULES` and never touched `session.runtime`, which is why a - # nurilab or gtmlab admin opened Manage user and saw Royal Imports' Customer and Product - # databases. `_perm_modules` applies the tenant's own provisioning (the rule `routes_nav` - # has applied since wave 18) and merges the tenant's own `ut_*` databases. - modules = _perm_modules(session, surfaces=True) - # ⭐⭐ W36-T22 / C2 — EVERY listed database, not an `enforced` subset of them. See - # `perms.tenant_governable_modules` for why the flag is deleted rather than defaulted. - governed = [m["key"] for m in modules] - perms_out = {} - for k in governed: - e = stored.get(k) - # ⛔⛔ W36-T22 — THE DEFAULT IS `may_read`, NOT `may_access`, AND THE DIFFERENCE IS AN - # OUTAGE. `may_access` reads migrated-and-undeclared as DENY. That is right for a registry - # topic and catastrophic for a `ut_*` key, because no `ut_*` entry was STORABLE before this - # wave — so every migrated account carries none, and this route would default every one of - # them to `access: false`. - # - # ⛔ AND IT WOULD NOT HAVE STAYED A DISPLAY BUG FOR LONG. C2 also deleted the filters that - # kept `ut_*` keys OUT of the PUT body, so the editor now sends every declared module: an - # administrator opening this page, changing one unrelated filter and pressing Save would - # write an EXPLICIT `{access: false}` for all ten keychain databases — at which point - # `may_read`'s deny-only overlay fires and revokes them for real. One click, silent, - # permanent. Defaulting from the SAME evaluator the read door uses is what makes the page - # show what the user can actually open, so a save of an untouched payload is a no-op. - # ⚠ `_public(uname, rec)`, NOT the raw record. A stored record is keyed BY username in - # `users.json` and does not carry one INSIDE it, so `may_read` would hand `may_open` a - # `None` viewer and get a fail-closed False — the very outage this line exists to prevent, - # arriving through the fix for it. The same trap cost a gate double an hour earlier today. - # ⛔⛔ A SURFACE KEY DEFAULTS FROM `nav_may_open`, NOT `may_read`, AND THE - # DIFFERENCE IS A MASS REVOCATION. `may_read` on a non-`ut_` key falls through to - # `may_access`, which reads migrated-and-undeclared as DENY — correct for a topic - # (every save writes topic entries) and false for a surface, because NO surface entry - # was storable before 2026-08-18, so every migrated account carries none. The editor - # would paint Assistant unchecked for an account that opens it every day, and the next - # save of ANY unrelated change would write the explicit deny for real. `nav_may_open` - # is the evaluator the nav and the route gate actually ask, legacy fallback included, - # so the box shows what the account can reach and an untouched save stays a no-op. - _default = (perm_scope.nav_may_open(users._public(uname, rec), k) - if any(m.get("surface") and m["key"] == k for m in modules) else - perm_scope.may_read(users._public(uname, rec), k, st=session.runtime)) - # ⭐⭐ W38-T19 — `metrics` RIDES EVERY ENTRY, BACKFILLED TO GRANTED ON A RECORD WRITTEN - # BEFORE THE KEY EXISTED. The wire is TOTAL on purpose: the client's own parse already - # reads an absent key as granted, so this line changes no rendering — what it changes is - # what a HUMAN reads off the payload while debugging, and what the PUT round-trips. The - # stored dict is copied rather than mutated: `rec` is the live registry record and - # stamping a key onto it here would write a permission nobody saved. - perms_out[k] = (dict(e, metrics=bool(e.get("metrics", True))) - if isinstance(e, dict) else - {"access": bool(_default), "filter": None, "hiddenFields": [], - "metrics": True}) - return {"username": uname, - "perms": perms_out, - # S3 ask 2 — the marker rides the GET. An absent module entry means DENY on a - # migrated record and LEGACY on an un-migrated one: opposite meanings for identical - # JSON, so the client cannot render honestly without knowing which world it is in. - "perms_v": int(rec.get("perms_v") or 0), - # `role == 'admin'` bypasses perms entirely (amendment 4) — sent so the editor can - # say "Everything (admin)" instead of rendering stored rules that do not apply. - "is_admin": perms.is_admin(rec), - # ⭐⭐ W36-T22 / C2 — `{key, label}` per row and NO `enforced`. It used to ride here - # so the editor could paint an apology ("Not set here") for a database whose wall was - # never applied; W36-T21 armed every one of them, so the flag would now be constantly - # true and the apology would be a lie with a green gate behind it. The LABEL comes - # from the registry / the table's own definition. - "modules": modules, - # S3 ask 1 — CANONICAL SHAPE: a BARE ARRAY of fields per module key. Not the nav - # schema envelope; the editor needs the field list and nothing else, and one shape - # beats two readings of a sentence. - # ⭐ EVERY governed database now carries its fields, `ut_*` included — that is owner - # item 11's *"EVERY database should be able to be toggleable by admin"*, and it is - # the SAME resolver `_clean_perms` validates against, so the picker cannot offer a - # column the validator will reject. - "fields_by_module": {k: _module_fields(k, session=session) for k in governed}} - - -@router.put("/admin/users/{username}/perms") -def put_perms(username: str, body: dict = Body(default=None), - session: Session = Depends(admin_gate)): - """Replace this account's permission block wholesale. - - WHOLESALE, not a merge: "remove this restriction" has to be expressible, and a merge cannot - express a deletion without a second vocabulary for it. - - ⚠ NO EPOCH BUMP, for the same reason `PATCH /admin/users/{u}` does not bump one: - `deps._user_for` re-reads the record on every request, so a narrowed wall applies to the - target's LIVE session on its very next call. Bumping would only add a forced re-login on top - of a change that has already taken effect. - """ - body = body if isinstance(body, dict) else {} - reg = _registry() - uname = str(username or "").strip().lower() - if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): - # Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one — - # 404, never 403, because "that account exists in another company" is itself a leak. - raise err(404, "no_such_user", f"no account named {uname!r}") - if "perms" not in body: - raise err(400, "empty_patch", "no perms to save") - - # THE SELF-LOCKOUT GUARD, the twin of `self_demote` on the PATCH route. An admin bypasses - # perm_scope entirely, so this cannot brick them today — but it stops an administrator - # writing a wall for themselves that would bite the moment their role changed. - if uname == session.uname: - raise err(400, "self_perms", - "you cannot set permissions on your own account — ask another admin") - - _mods = _perm_modules(session, surfaces=True) - # ⭐⭐ W36-T22 / C2 — ONE key set. It was two ("listed" and "enforced") with a refusal in the - # gap; W36-T21 closed the gap, so every database the editor shows is one whose wall the table - # routes apply. `session` rides so a `ut_*` schema can be read for THIS tenant. - cleaned = _clean_perms(body.get("perms"), - governed_keys={m["key"] for m in _mods}, session=session) or {} - cleaned = _fold_legacy_scope(reg[uname], cleaned) - _guard_bu_widening(reg[uname], cleaned, bool(body.get("confirm_widen"))) - users.set_access(uname, perms=cleaned) - fresh = _registry() - rec = fresh.get(uname) or {} - if int(rec.get("perms_v") or 0) < users.PERMS_VERSION: - # The store took the write and did not record it. A 200 here would tell an administrator - # the wall is up when it is not — the one direction this must never fail in. - raise err(503, "store_unavailable", - "the permissions were not saved — the account is UNCHANGED") - return {"username": uname, "perms": rec.get("perms") or {}, - "perms_v": int(rec.get("perms_v") or 0)} - - -def _store_binding(session): - """`data_binding.describe()` for the store THIS SESSION's tenant actually writes. - - ⭐ THE TENANT'S STORE, NOT THE PROCESS DEFAULT, AND THAT DISTINCTION IS THE POINT. Tenant #0 - rides `core.store`'s module default, but nurilab/gtmlab/loopable are bound to their OWN dataset - repos by `harness/runtime.py:470`, straight out of the control-plane record — which is the path - that carried three tenants' PRODUCTION stores onto staging while `--data-repo` isolated only - tenant #0. Reporting the process default here would show "staging store, all fine" to exactly - the tenants that were not fine. - - ⚠ Degrades to a stated `unknown` rather than raising: a settings page that 500s because it - could not describe the store is worse than one that says it does not know. - """ - try: - import core.data_binding as data_binding # noqa: PLC0415 - import core.store as store # noqa: PLC0415 - bound = getattr(session.runtime, "store_handle", None) - repo = getattr(bound, "repo", None) or store.REPO - return data_binding.describe(repo) - except Exception as e: # noqa: BLE001 - return {"deployment": "unknown", "production": False, "repo": "", - "writable": False, "refusal": f"could not resolve the store binding: {e}", - "localOverride": False, "allowlist": [], "observed": {}} - - -@router.get("/settings") -def settings(session: Session = Depends(require_session)): - """The session's OWN settings — any authenticated user, not admin-only. - - `user` comes from `routes_auth._public_user`, deliberately reusing the ONE definition of what a - client may know about itself (never a hash, never the epoch). `scope` restates it as the two - things the Settings UI shows, and `tenant` names only THIS tenant — enumerating the others - would answer a question about our customer list. - """ - modules = perms.allowed_modules(session.user) - return { - # The build this Space is running. Behind the session on purpose — see `main.py::health`, - # which refuses it for being the first URL a scanner finds. `deploy_web.py` stamps it as a - # Space secret at every deploy; absent means "somebody started this container by hand". - "version": os.environ.get("AIOS_VERSION") or "unknown", - "user": _public_user(session.user), - "scope": { - "bus": perms.allowed_bu_labels(session.user), - "team_id": perms.scope_team_id(session.user), - "agent": perms.scope_agent(session.user), - "modules": sorted(modules) if modules is not None else "all", - # C-PERM: the caller's OWN effective wall, read-only. The grid uses it to grey - # pickers rather than to enforce — hidden fields are stripped from every data wire - # regardless, so a client that ignores this is narrowed anyway, never widened. - "perms": session.user.get("perms") or {}, - }, - "tenant": {"key": session.tenant, "name": getattr(session.runtime, "name", - session.tenant)}, - # ⭐⭐ D-315 — WHICH STORE THIS CONTAINER IS BOUND TO, AND WHETHER IT MAY WRITE IT. - # - # ⛔ THIS IS NOT DIAGNOSTIC GARNISH; IT IS THE ONLY WAY THE QUESTION CAN BE ANSWERED. - # D-160 established that a Space's environment cannot be read from outside, so "is staging - # pointed at production data?" is a question only the container can answer — and it went - # unanswered for the whole window in which two builds wrote tenant #0's real store. - # `verify_live` asserts this field after logging in, which is a check that survives a role - # swap in a way that reading a deploy log never did. - # - # ⚠ Session-gated, beside `version`, for `main.py::health`'s reason: a Space id and a - # dataset repo id are public names, but a health endpoint is the first URL a scanner finds - # and it may not describe the deployment. No customer data is here — only which store, and - # yes or no. - "store": _store_binding(session), - # CHROME ONLY. Every /admin route re-checks the role server-side; this exists so the client - # does not paint a "Manage users" button that 403s. - "admin": perms.is_admin(session.user), - # Wave 19 (R3, contract C2): the same courtesy for the LOOPABLE plane — the Settings rail - # paints its entry iff this is true, and `routes_platform_admin` refuses every request - # that is not, so a client that ignored this would gain exactly nothing. Derived from the - # ONE predicate rather than restated: a second copy of a two-condition wall is a second - # place for one of the conditions to go missing. - "platformAdmin": platform_admin.is_platform_admin(session.user), - } +"""routes_admin.py — Y4: user administration + the session's own settings (W2-3). + +These routes are the standalone shell's replacement for `app.py`'s `users_dialog` / `settings_dialog` +modals, over the SAME `core/users` account store — so both front-ends administer one set of +accounts and the eventual OIDC migration (D-3) swaps the CREDENTIAL check, not the user model. + +ADMIN-ONLY, FAIL-CLOSED. `admin_gate` is a role check (`perms.is_admin`), mirroring the Streamlit +dialog's `if not is_admin()`. A default record has `role: 'user'`, so an unreadable or partial record +is denied rather than admitted. `verify_api.py` proves it by having a viewer TRY every route. + +⛔ THIS IS THE FIRST WRITER OF `modules` THAT HAS EVER EXISTED. `core.users.set_access(modules=…)` +has no caller anywhere in the shipped app — module grants are only settable by editing the store +JSON out of band. That matters because the record format carries TWO documented fail-OPENS, both +asserted in `verify_api.py` section A: + + * `modules: []` is FALSY, so `perms.allowed_modules` reads it as 'all' = UNRESTRICTED. An admin + clearing every checkbox to lock an account down would grant it everything. + * `bus: []` falls through `allowed_bus_labels`'s "no recognisable label" branch to + `['All','Fisch','Royal']` — so one typo'd BU id is FULL cross-BU access, in the model whose + whole point is strict isolation. + +Y4 says do not "fix" `modules: []` in this wave, and that is right: the READER is mirrored in +`ui/session.py` and `core/perms.py` and diverging one of them mid-wave breaks lock-step. But the +WRITER is new, and it can simply refuse to create either footgun. So both are 400s here and both +read semantics are untouched — the seam is write-strict / read-unchanged. + +⚠ A STORE OUTAGE IS A 503, NEVER AN EMPTY LIST. `users.registry()` swallows a failed read into `{}` +one level down, and serving that as `{"users": []}` would tell an administrator their tenant has no +accounts. Same rule as the write path: an empty 200 is never how this API says "something is wrong". +""" +import os +import re + +from fastapi import APIRouter, Body, Depends, Response + +import core.platform_admin as platform_admin # wave 19 R3 — the /settings chrome flag +import core.registry as registry +import core.store as store + +from deps import Session, err, perms, require_session, users +from routes_auth import _public_user + +router = APIRouter(prefix="/api/v1") + +#: A username is a STORE KEY (`users.json`) and also the key the table workspace is filed under +#: (`data[username]`), so it is constrained rather than trusted: lowercase, no separators, no +#: whitespace, nothing that could traverse or collide once it becomes part of a path or a filename. +_UNAME_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{1,31}$") +#: ⭐⭐ WAVE 37 · T03 (owner ruling R8) — AN EMAIL ADDRESS IS ALSO A USERNAME. Owner: the username +#: IS the email, one identity. ⛔ THIS IS NOT A NEW CAPABILITY, IT IS THE DOOR CATCHING UP WITH THE +#: DATA: measured on deployed staging before the change, FIVE of tenant #0's eight accounts already +#: carry an `@` in the `username` field (`david@fischfloralsupply.com`, `farhan@`, `florencia@`, +#: `karen@`, `naomi@`) — every one of them illegal under the slug pattern above and longer than its +#: 32-character cap, so this route could not have created any of them. +#: ⚠ DELIBERATELY PERMISSIVE, because the store key is the thing being constrained and not the +#: deliverability of the address: one `@`, a dot-bearing domain, and the same character class the +#: slug already trusts. A stricter grammar here would refuse addresses that exist. +_UNAME_EMAIL_RE = re.compile(r"^[a-z0-9][a-z0-9._%+-]*@[a-z0-9][a-z0-9.-]*\.[a-z]{2,24}$") +#: RFC 5321's cap on a whole address. The slug form keeps its own 32 via `_UNAME_RE`. +_UNAME_MAX = 254 +#: Passwords are PBKDF2-200k, which is only as strong as what it is given. The Streamlit dialog +#: enforces nothing; this does, and the two are allowed to differ because only one of them is +#: reachable from the internet. +_MIN_PW = 8 +_ROLES = ("user", "admin") + + +def admin_gate(session: Session = Depends(require_session)) -> Session: + """401 without a session, 403 without the admin role. Role, not a module grant — mirroring + `app.py`'s `users_dialog`, whose only wall is `is_admin()`.""" + if not perms.is_admin(session.user): + raise err(403, "forbidden", "administrators only") + return session + + +def _require_store(): + """A 503 the moment the store cannot serve, so no route below can report emptiness as truth.""" + try: + ok = store.available() + except Exception: + ok = False + if not ok: + raise err(503, "store_unavailable", + "the tenant store is unavailable — accounts cannot be read or changed") + + +def _registry(): + _require_store() + try: + reg = users.registry() or {} + except Exception: + raise err(503, "store_unavailable", "the tenant store is unavailable") + return reg + + +def _view(uname, rec, governed=None, st=None, surface_keys=None): + """What an administrator may see about an account. NEVER `salt` or `hash` — this function is + the only projection these routes use, so there is one place that can leak and it does not. + + ⚠ `governed`/`st` ride through to `_access_summary` and nowhere else. The ROSTER passes both, + resolved once for the whole request; the single-record routes pass neither, because their + caller fetches the editor payload separately and `GET /perms` is the authority there. + """ + return {"username": uname, + "name": rec.get("name") or uname, + "role": rec.get("role", "user"), + "bus": rec.get("bus", "all"), + "bu_labels": perms.allowed_bu_labels(rec), + "modules": rec.get("modules", "all"), + "agent": rec.get("agent") or None, + "email": rec.get("email") or None, + "tenant": _tenant_of(rec), + "active": bool(rec.get("active", True)), + # The session-revocation handle. An admin needs it: it is the only visible evidence + # that "sign them out everywhere" actually happened. + "epoch": int(rec.get("epoch") or 0), + # S3 ask 4 — one sentence for the roster's Access column, so the list does not need + # a /perms round trip per row to fill one cell. Computed from the same record the + # editor will open, so the two cannot disagree. + "accessSummary": _access_summary(rec, uname=uname, governed=governed, st=st, + surface_keys=surface_keys), + "perms_v": int(rec.get("perms_v") or 0)} + + +def _access_summary(rec, uname="", governed=None, st=None, surface_keys=None): + """One sentence describing what this account may reach. Deliberately says LESS than the + editor: it is a signpost, not a rule listing, and a summary that tried to spell out filters + would be wrong the moment a filter got interesting. + + ⛔⛔ W36-T22 — IT MUST COUNT WHAT THE EDITOR WOULD SHOW, NOT WHAT IS STORED, AND THE TWO STOPPED + AGREEING THE MOMENT `get_perms` LEARNED TO DEFAULT FROM `may_read`. This counted STORED ENTRIES + only, which WAS the same answer before this wave: a migrated account had an entry for every + governable database, because the editor wrote one per topic on every save and no `ut_*` entry + could exist at all. Now a `ut_*` key legitimately has NO entry and is still OPEN, so the roster + would say *"1 module"* about an account the editor shows reaching twelve. ⛔ The roster is the + screen an administrator reads FIRST, and two screens disagreeing about one account is the + defect this wave exists to remove, not a rounding difference. + + ⚠ `governed`/`st` ARE PASSED IN RATHER THAN DERIVED HERE, and that is a cost decision. This + runs once PER ROW of the roster; asking `may_read` per (account x database) without a lend + would be a whole-document copy per question — D-213's shape, one route over. The caller + resolves the tenant's list and ONE lend for the whole request. A caller that passes neither + keeps the stored-entry reading, which is right for the single-record routes: their client + fetches `GET /perms` separately and that payload is the authority. + """ + import core.perm_scope as perm_scope + + if perms.is_admin(rec): + # amendment 4: an admin bypasses `perms` entirely, so rendering their stored rules as if + # they applied is the one misreading of that clause that could actually hurt. + return "Everything (admin)" + if not perm_scope.is_migrated(rec): + mods = perms.allowed_modules(rec) + return "All modules (legacy rules)" if mods is None else \ + f"{len(mods)} module{'' if len(mods) == 1 else 's'} (legacy rules)" + entries = (rec.get("perms") or {}) + if governed: + principal = users._public(uname, rec) if uname else rec + # ⛔ PER KIND, MIRRORING `get_perms`' OWN DEFAULTS — the first draft asked + # `nav_may_open` for EVERY key and that was a second idea of the count: for a migrated + # record with no topic entry the editor's default is `may_read` (DENY), so a blanket + # admission read would make the roster say "3 modules" about an account whose editor + # shows one box ticked. The roster's whole contract (leg 6 of the T22 section) is that + # it counts what the editor would show; a SURFACE key defaults from admission there, + # everything else from `may_read`, so this does exactly the same, key by key. + opened = [k for k in governed + if (perm_scope.nav_may_open(principal, k, st=st) + if k in (surface_keys or ()) else + perm_scope.may_read(principal, k, st=st))] + else: + opened = [k for k, e in entries.items() if isinstance(e, dict) and e.get("access", True)] + restricted = sum(1 for k in opened + if isinstance(entries.get(k), dict) + and (entries[k].get("filter") or entries[k].get("hiddenFields"))) + if not opened: + return "No modules" + base = f"{len(opened)} module{'' if len(opened) == 1 else 's'}" + return f"{base}, {restricted} restricted" if restricted else base + + +# ── request validation: every rule below is fail-closed ────────────────────────────────────────── +def _clean_username(v): + """The ONE validator for a new account's key. R8 widened it to admit a full email address. + + ⛔ IT HAS EXACTLY ONE CALL SITE AND THAT IS LOAD-BEARING, not an oversight to tidy up. + `PATCH /admin/users/{username}`, the password route and both `/perms` routes take the username + as a PATH parameter and never re-validate it, which is precisely why the five email-keyed + accounts above are administrable today despite being unconstructable. Arming this validator on + those routes would look like hardening and would break five live accounts. + """ + uname = str(v or "").strip().lower() + if len(uname) > _UNAME_MAX or not (_UNAME_RE.match(uname) or _UNAME_EMAIL_RE.match(uname)): + raise err(400, "bad_username", + "a username is either 2 to 32 characters of lowercase letters, digits, dot, " + "dash or underscore, or a full email address of up to 254 characters") + return uname + + +def _clean_password(v): + pw = str(v or "") + if len(pw) < _MIN_PW: + raise err(400, "weak_password", f"a password must be at least {_MIN_PW} characters") + return pw + + +def _clean_role(v): + role = str(v or "").strip().lower() + if role not in _ROLES: + raise err(400, "bad_role", f"role must be one of {list(_ROLES)}") + return role + + +def _clean_bus(v): + """'all', or a non-empty list of KNOWN business-unit ids. + + Refusing an unknown id is the whole point: `allowed_bus_labels` treats a list with no + recognisable label as full access, so `bus: [7]` would be a cross-BU grant created by a typo. + """ + if isinstance(v, str): + if v.strip().lower() == "all": + return "all" + raise err(400, "bad_bus", "bus must be 'all' or a list of business-unit ids") + if not isinstance(v, (list, tuple)) or not v: + raise err(400, "bad_bus", + "bus must be 'all' or a NON-EMPTY list of business-unit ids (an empty list " + "would read as unrestricted)") + out = [] + for b in v: + try: + b = int(b) + except (TypeError, ValueError): + raise err(400, "bad_bus", "a business-unit id must be a number") + if b not in users.BU_LABELS: + raise err(400, "bad_bus", + f"{b} is not a known business unit {sorted(users.BU_LABELS)} — an " + "unrecognised id would widen access to every BU") + out.append(b) + return sorted(set(out)) + + +def _clean_modules(v): + """'all', or a non-empty list of KNOWN module keys. + + `[]` is refused because it READS as unrestricted (see the module docstring). An unknown key is + refused because `may_open` is fail-closed on it: an admin who typed `sale` for `sales` would + silently lock the account out of the page they meant to grant. + """ + if isinstance(v, str): + if v.strip().lower() == "all": + return "all" + raise err(400, "bad_modules", "modules must be 'all' or a list of module keys") + if not isinstance(v, (list, tuple)) or not v: + raise err(400, "bad_modules", + "modules must be 'all' or a NON-EMPTY list of module keys — an empty list " + "reads as UNRESTRICTED, which is the opposite of locking an account down") + known = set(registry.BY_KEY) | set(perms._LEGACY_KEYS) + out, bad = [], [] + for k in v: + k = str(k or "").strip() + (out if k in known else bad).append(k) + if bad: + raise err(400, "bad_modules", f"unknown module keys: {sorted(bad)}") + return sorted(set(out)) + + +def _tenant_of(rec): + """The account's company, with the pre-wave default. ONE spelling of the default, used by + every admin route — two spellings is how a tenant wall grows a gap.""" + return str((rec or {}).get("tenant") or "royal-imports").strip().lower() + + +# ── the routes ─────────────────────────────────────────────────────────────────────────────────── +@router.get("/admin/users") +def list_users(session: Session = Depends(admin_gate)): + """Wave 18 (C1-TENANT): scoped to the CALLER'S tenant. The user registry is a global + control-plane bucket, so without this filter a Nurilab admin would read Royal's whole + roster — names, emails, agents — from one URL.""" + reg = _registry() + mine = _tenant_of(session.user) + # ⭐ W36-T22 — the tenant's governable list and ONE lend, resolved once for the whole roster so + # `_access_summary` can agree with the editor without paying a document copy per question. + import core.user_tables as _ut + try: + _rows = _perm_modules(session, surfaces=True) + governed = [m["key"] for m in _rows] + surface_keys = {m["key"] for m in _rows if m.get("surface")} + lent = _ut.lend_defs(session.runtime) + except Exception: # noqa: BLE001 + # A store that cannot list the tenant's databases must not take the roster down. The + # summary degrades to the stored-entry reading, which is what it always was. + governed, lent, surface_keys = None, None, None + return {"users": [_view(u, reg[u], governed=governed, st=lent, surface_keys=surface_keys) + for u in sorted(reg) + if _tenant_of(reg[u]) == mine]} + + +@router.post("/admin/users", status_code=201) +def create_user(body: dict = Body(default=None), session: Session = Depends(admin_gate)): + """Create an account. **409 if the username already exists — `PATCH` is the update path.** + + ⛔ WHY 409 AND NOT AN UPSERT. `core.users.create_user` writes a fresh `_record()`, and a fresh + record has NO `epoch` — so overwriting an existing account used to drop its epoch back to 0 and + RESURRECT every cookie minted before its last password change. `core/users.py` now + preserves-and-bumps on overwrite (which also closes it for `app.py`'s dialog, where "Add / + update a user" calls the same function), but this route still refuses: an upsert that silently + replaces an account's password, role and BU access because someone reused a username is not an + update anybody asked for. + """ + body = body or {} + _require_store() + uname = _clean_username(body.get("username")) + pw = _clean_password(body.get("password")) + role = _clean_role(body.get("role") or "user") + bus = _clean_bus(body.get("bus") if body.get("bus") is not None else "all") + modules = _clean_modules(body.get("modules") if body.get("modules") is not None else "all") + name = str(body.get("name") or "").strip() or uname + agent = str(body.get("agent") or "").strip() or None + email = str(body.get("email") or "").strip() or None + + _reg_before = _registry() + if uname in _reg_before: + raise err(409, "user_exists", f"{uname} already exists — PATCH it to change it") + # ⭐⭐ WAVE 37 · T03 (R8) — THE COLLISION THE WIDENING CREATES, REFUSED AT CREATE TIME. + # `core/users.py::verify` resolves a typed string by looking up the registry KEY first and only + # scanning the `email` field when that misses (its own wave-18 R1 comment). That precedence was + # incidental while a username could not contain `@`; now it decides which of two accounts a + # person reaches, so it is stated in both files rather than left to be re-derived. + # ⛔ BOTH DIRECTIONS, because each one silently takes a login away from an account that has it: + # (a) a new USERNAME equal to somebody's EMAIL wins the key lookup and shadows their address; + # (b) a new EMAIL equal to somebody's USERNAME can never be reached, because their key wins. + # Refused with a named error rather than allowed and explained afterwards: the failure is a + # person signing in and landing in the wrong account, which nothing downstream can detect. + if "@" in uname: + _shadowed = next((k for k, r in _reg_before.items() + if isinstance(r, dict) + and str(r.get("email") or "").strip().lower() == uname), None) + if _shadowed: + raise err(409, "username_shadows_email", + f"{uname} is already the email address on the account {_shadowed}, which " + f"signs in with it today. This account would take that login over. Choose a " + f"different username, or clear the email address on {_shadowed} first.") + if email: + _e = email.strip().lower() + if _e != uname and _e in _reg_before: + raise err(409, "email_shadows_username", + f"{_e} is already a username on this deployment, so anyone signing in with " + f"it reaches that account and never this one. Choose a different email " + f"address for this account.") + # Wave 18 (C1-TENANT): an admin creates accounts in their OWN company, never another's — + # the tenant is stamped from the session, not taken from the body. + users.create_user(uname, pw, name, role=role, bus=bus, modules=modules, + agent=agent, email=email, tenant=_tenant_of(session.user)) + reg = _registry() + if uname not in reg: + # The store accepted the write and does not have it. Reporting 201 here would be the + # "200 over a write that evaporated" failure the whole store-outage rule exists to prevent. + raise err(503, "store_unavailable", "the account was not saved — try again") + return {"user": _view(uname, reg[uname])} + + +@router.patch("/admin/users/{username}") +def update_user(username: str, body: dict = Body(default=None), + session: Session = Depends(admin_gate)): + """Change access on an existing account. Only the keys PRESENT in the body change. + + ⚠ NO EPOCH BUMP, and that is not an omission. `deps._user_for` re-reads the record on every + request, so a narrowed role / BU / module grant applies to the target's LIVE session on its very + next call — bumping the epoch would only add a forced re-login on top. `active: false` is the + exception and it bumps, inside `core.users.set_active`, because a disabled account must lose its + session rather than keep working until the cookie expires. + """ + body = body if isinstance(body, dict) else {} + reg = _registry() + uname = str(username or "").strip().lower() + if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): + # Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one — + # 404, never 403, because "that account exists in another company" is itself a leak. + raise err(404, "no_such_user", f"no account named {uname!r}") + if not body: + raise err(400, "empty_patch", "no fields to update") + + unknown = sorted(set(body) - {"name", "role", "bus", "modules", "active", "agent", "email"}) + if unknown: + # A silently-ignored field is how a UI ends up believing it saved something it did not. + raise err(400, "unknown_fields", f"cannot update: {unknown}") + + is_self = uname == session.uname + role = _clean_role(body["role"]) if "role" in body else None + active = bool(body["active"]) if "active" in body else None + # THE ONE-CLICK LOCKOUT GUARD. APP_PASSWORD remains the real backstop for 'admin', so this is + # not the thing that keeps the product reachable — it just stops an administrator removing + # their own access with a toggle and having to go find the master password. + if is_self and role is not None and role != "admin": + raise err(400, "self_demote", + "you cannot remove your own administrator role — ask another admin") + if is_self and active is False: + raise err(400, "self_deactivate", "you cannot deactivate your own account") + + kw = {} + if role is not None: + kw["role"] = role + if "name" in body: + kw["name"] = str(body["name"] or "").strip() or uname + if "bus" in body: + kw["bus"] = _clean_bus(body["bus"]) + if "modules" in body: + kw["modules"] = _clean_modules(body["modules"]) + if "agent" in body: + kw["agent"] = str(body["agent"] or "").strip() # '' clears the link + if "email" in body: + kw["email"] = str(body["email"] or "").strip() + if kw: + users.set_access(uname, **kw) + if active is not None and active != bool(reg[uname].get("active", True)): + users.set_active(uname, active) # bumps the epoch: kills live sessions + + fresh = _registry() + return {"user": _view(uname, fresh[uname])} + + +@router.post("/admin/users/{username}/password", status_code=204) +def set_password(username: str, body: dict = Body(default=None), + session: Session = Depends(admin_gate)): + """Rotate a password. **Revokes every outstanding session for that account** — the epoch bump + happens inside the same read-modify-write as the hash (`core.users.set_password`), so the two + can never disagree. + + 204 with no body: there is nothing to say that the caller did not already know, and echoing the + account back would invite a client to diff a response for a password change. + """ + reg = _registry() + uname = str(username or "").strip().lower() + if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): + # Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one — + # 404, never 403, because "that account exists in another company" is itself a leak. + raise err(404, "no_such_user", f"no account named {uname!r}") + pw = _clean_password((body or {}).get("password")) + before = int(reg[uname].get("epoch") or 0) + users.set_password(uname, pw) + after = int((_registry().get(uname) or {}).get("epoch") or 0) + if after <= before: + # The whole value of this route is the revocation. If the epoch did not move, the sessions + # were not revoked, and answering 204 would say they were. + raise err(503, "store_unavailable", + "the password change did not persist — outstanding sessions were NOT revoked") + return Response(status_code=204) + + +# ── C-PERM (wave 15): per-module access + permanent filters + hidden fields ────────────────── +#: The modules the permission editor governs — the GRID surfaces, the ones with a field schema +#: to filter and hide. Amendment 6 shipped the wall on `customer_data` alone and said this list +#: is the one place that changes when C-TOPIC lands. WAVE 16: it landed, so `product_data` +#: joins — and it is not cosmetic. `perms_v: 1` means an UNDECLARED module DENIES (amendment 4), +#: so a migrated account would be locked out of Product with no control anywhere to grant it: +#: fail-closed, but unadministrable. The editor grows its section with zero client edits +#: (S3 built it off this route's `modules` + `fields_by_module`). +#: +#: ⭐⭐ WAVE 33 (owner item 11, W33-T33) — **THIS IS A CATALOGUE, NOT AN ANSWER.** It used to be +#: both, and that was the defect the owner flagged four times: `get_perms` served this tuple +#: verbatim and never touched `session.runtime`, so EVERY tenant was handed tenant #0's two grid +#: modules. What this names now is a fact about THIS FILE — the registry topics `_module_fields` +#: below has a field-schema arm for — and the answer a tenant receives is `_perm_modules(session)`, +#: which filters this by the tenant's own provisioning and merges the tenant's own `ut_*` +#: databases. ⛔ Serving this list directly again re-opens item 11; `verify_perm_scope`'s +#: `section_tenant_derived` NC does exactly that and reds. +#: ⭐ CONTRACT C3, CONSUMED (lane E's `W33-T46`, answered as `E-3`). The topic list is DERIVED from +#: `registry.governable_modules()` — `{topic: {'key','label','subject'}}` over every non-archived, +#: non-group_only registry row that declares a `topic`. What stays hand-written is +#: `_FIELD_PROVIDER_KEYS` below, and that is CODE, not policy: each name has its own `aios_grid` +#: contract behind it in `_module_fields`, and a topic with no arm cannot be governed by this +#: editor at all. The intersection is the honest answer to "what can this module build pickers for". +#: +#: ⚠ KEPT AS A SYMBOL RATHER THAN DELETED, deliberately. `verify_api`'s W30a reads +#: `routes_admin._PERM_MODULES` to assert `perm_scope.BU_FILTERABLE_MODULES` names exactly the +#: governed topics whose contract carries `dba`. Deleting the name would make that gate raise +#: `AttributeError` — a CRASH, which scores as "the gate is broken" rather than as a red +#: [[gate-must-go-red-not-crash]] — and `verify_api.py` is lane A's fence. Derived-and-kept gives +#: that assertion a truer subject than the literal ever did, and costs nobody a cross-fence edit. +_FIELD_PROVIDER_KEYS = frozenset({"customer_data", "product_data"}) + + +def _governable_catalogue(): + """The REGISTRY topics this module can govern: `[{key, label}]`, in registry order. + + ⛔ No tenant filtering here — that is `perms.tenant_governable_modules`' job, and E's map is + tenant-blind by design. Two filters in two files answering one question is how the perms route + and `routes_nav` came apart in the first place. + """ + import core.registry as registry + return [{"key": v["key"], "label": v["label"]} + for v in registry.governable_modules().values() + if v.get("key") in _FIELD_PROVIDER_KEYS] + + +_PERM_MODULES = tuple(m["key"] for m in _governable_catalogue()) + + +def _perm_modules(session, surfaces=False): + """The databases THIS tenant's permission editor governs: `[{key, label, enforced}]`. + + Three inputs, none of them a literal: the topic catalogue above, the tenant's provisioned + module set (`tenant.config['modules']`, exactly the rule `routes_nav.py::nav` applies), and + the tenant's own `ut_*` databases read through the DEFINITIONS projection. + + ⭐ `user_tables.nav_entries` rather than a fresh listing, deliberately: it is already the ONE + resolver for "which databases may this account see", it is `may_open`-filtered, and it reads + the projected document (`all_defs`) so this costs definitions, never the 28.6 MB of rows. + The caller is admin-gated, so the filter admits the whole tenant — but asking the resolver + instead of assuming that is what keeps this from becoming the SECOND idea of who may see a + table (`user_tables.may_open`'s own docstring is the record of the first time that happened). + """ + import core.perms as _perms + import core.user_tables as user_tables + try: + ut = user_tables.nav_entries(viewer=session.uname, is_admin=True, + st=session.runtime) + except Exception: + # A store that cannot list the tenant's databases must not take the whole editor down — + # the topic half is still administrable, and an empty `ut_*` half is visibly empty. + ut = () + cat = _governable_catalogue() + skeys = set() + if surfaces: + # ⭐⭐ OWNER RULING 2026-08-18 — the Assistant and Agents surfaces are + # governable ACCESS toggles in the ACCOUNT editor. `surfaces` is caller-explicit and + # defaults CLOSED because the other consumer of this list is the CHANNEL-AGENT editor + # (`routes_slack.get_channel_agent`), whose principal is enforced per DATABASE + # (`agent_rows`/`agent_may_read`) and never opens an app surface: an Assistant toggle + # there would be a stored rule nothing applies, the exact class C2 deleted. + surf = _perms.governable_surface_modules() + skeys = {s["key"] for s in surf} + cat = cat + surf + rows = _perms.tenant_governable_modules(session.runtime, cat, ut_entries=ut) + # The marker survives OUTSIDE `tenant_governable_modules`, which builds bare {key,label} + # rows: the client needs it only to choose the schemaless help copy, and the server needs + # it only to pick the admission default in `get_perms`. + return [dict(r, surface=True) if r["key"] in skeys else r for r in rows] + + +#: ⛔ `_enforced_keys` IS DELETED (W36-T22 / CONTRACT C2). It asked `perms.enforced_module_keys` +#: which subset of the listed databases a wall could actually be stored against, and after R6 the +#: answer is "all of them" — a question with one possible answer is not a question. Both it and the +#: `perms` helper behind it are gone rather than made to return everything, because a predicate +#: that ignores its argument is the always-true flag C2 deletes, one indirection along. +#: What callers use instead is `{m["key"] for m in _perm_modules(session)}` — the SAME list the +#: editor renders, so "shown" and "storable" cannot drift apart again. + + +def _module_fields(key, session=None): + """The field schema the editor builds its pickers from — the SAME vocabulary the grid uses, + so an admin filters on exactly what the user sees. + + ⛔ Returned WITHOUT any per-user hiding applied, deliberately: this is the admin choosing + what to hide, so hiding the choices would make a field unhideable the moment it was hidden + once for somebody. The route is admin-gated; the projection a MEMBER receives is the one + `perm_scope.visible_fields` narrows. + + ⚠ PER TOPIC (wave 16). `product_data` has its OWN canonical contract, and handing the + editor the customer schema for it would let an admin build a wall out of columns that + module does not have — `_clean_perms` would then 400 on the admin's own choices, or worse, + a filter naming a field the product rows lack would DENY every row via `permits()`. + The product contract is served CONSOLIDATED (every column) on purpose: a BU-scoped reader + is served fewer columns, but the ADMIN is choosing what may ever be hidden. + + ⛔⛔ WAVE 33 — THE `else` USED TO BE A FALL-THROUGH, AND IT WAS SILENT-WRONG. The body read + `aios_grid.product_fields() if key == "product_data" else aios_grid.FIELDS`, so ANY key that + was not `product_data` got the CUSTOMER contract: the moment the governed list stopped being + a two-element literal, a `ut_*` database or a third topic would have been handed customer + columns, `_clean_perms` would have validated an admin's `hiddenFields` against them, and the + stored wall would name columns that module does not have — which `permits()` resolves by + DENYING every row. No error at any point. So the map is explicit and an unknown key RAISES: + a topic without a schema arm is a missing arm, never a customer grid in disguise. + + ⭐⭐ W36-T22 / R6 — AND `ut_*` NOW HAS AN ARM, WHICH IS THE HALF THAT MAKES THE TOGGLE REAL. + Owner item 11: *"how come the database is only toggleable for Odoo customers and Odoo + products … EVERY database should be able to be toggleable by admin."* A `ut_*` database's + field contract is its OWN stored definition, so the picker offers exactly the columns that + database has. ⛔ ONE SOURCE, NOT TWO: the picker (`get_perms.fields_by_module`) and the + validator (`_clean_perms`) both call this, so a column the picker offers can never be a + column the validator rejects. Getting that wrong is worse than an error message — + `permits()` DENIES on a predicate it cannot answer, so a mismatched wall is a + deny-everything trap wearing a saved-successfully toast. + + ⚠ `session` IS REQUIRED FOR A `ut_*` KEY and unused for a topic: a stored definition is + per-tenant data and there is no tenant-blind way to read one. A caller that omits it is + refused LOUDLY rather than handed another tenant's schema — the same rule the `else` + fall-through above was deleted for. + """ + import aios_grid + key = str(key or "") + if key in {s["key"] for s in perms.governable_surface_modules()}: + # ⭐ A SURFACE (Assistant, Agents) has NO field schema, and `[]` is the honest + # answer rather than the wave-26 empty-option trap: the editor's schemaless rendering + # is a deliberate state (access toggle only, R9), and `_clean_perms` refuses any + # filter or hiddenFields validated against this empty vocabulary. An unknown key + # still RAISES below — this arm admits exactly the derived surface list. + return [] + if key.startswith("ut_"): + if session is None: + raise err(400, "ungoverned_module", + f"{key!r} is a tenant database, so its columns can only be read for a " + f"known tenant — this caller passed no session") + import core.user_tables as user_tables + defn = user_tables.get(key, st=user_tables.lend_defs(session.runtime)) + if not defn: + raise err(400, "ungoverned_module", + f"{key!r} is not a database in this workspace") + src = defn.get("fields") or [] + else: + providers = {"customer_data": lambda: aios_grid.FIELDS, + "product_data": aios_grid.product_fields} + provider = providers.get(key) + if provider is None: + # Reached only by a caller that skipped `_perm_modules`. Loud, because the alternative + # (`[]`) is an empty option list read as an ANSWER — wave 26 item 24, `if ([])` is + # truthy. + raise err(400, "ungoverned_module", + f"{key!r} has no permission-editor field schema. The editor governs " + f"{sorted(providers)} and this tenant's own databases") + src = provider() + known = _server_side_vocabularies() + out = [] + for f in src: + if not isinstance(f, dict) or not f.get("key"): + continue + opts = f.get("options") + if not isinstance(opts, list) or not opts: + # ⭐ D-76 / W33-T35 — SUPPLY WHAT THE SERVER ALREADY KNOWS. See + # `_server_side_vocabularies`. + opts = known.get(f["key"]) + out.append({"key": f["key"], "label": f.get("label") or f["key"], + "type": f.get("type") or "text", + **({"options": list(opts)} if isinstance(opts, list) and opts else {}), + **({"pinned": True} if f.get("pinned") else {})}) + return out + + +def _server_side_vocabularies(): + """`{field key: [choice, …]}` for choice columns whose vocabulary is CLOSED and known here, + but which the grid contract does not declare. + + ⛔ D-76 — WHY THE PERMISSION EDITOR'S DROPDOWNS WERE EMPTY, AND WHY THE FIX IS HERE. The grid + resolves a choice column through `types.ts::choiceVocabulary`: the DECLARED list when the + field has one, otherwise the values seen in the loaded ROWS. This editor loads no rows, so + every column that leans on the second branch renders `Select…` with nothing in it — the + identical shape wave 26 item 24 closed on the Product grid, one surface over. ⚠ And the fix + belongs at the CALLER, never in the panel: `FilterBuilderPanel::choicesFor` treats a supplied + list as authoritative *including when it is empty* ("this column has no values" is an answer), + which is correct and must not be weakened. So the caller stops handing it nothing. + + ⭐ IMPORTED, NEVER RESTATED. `stock_bucket`'s labels are `modules/inventory.COVERAGE_LABELS`, + the same constant `_bucket()` mints from — a copy here would be a second vocabulary that + drifts the first time a band is renamed [[constant-two-features-share]]. The import is lazy + for the same reason `aios_grid`'s is: this route should not pull the analytics stack at + module load. + + ⚠ TWO COLUMNS ARE DELIBERATELY ABSENT, and R6's second sentence says to name them rather than + let them look handled: + · `category` (product) — an OPEN vocabulary read off Odoo's product categories. There is no + closed list to declare, and discovering it would mean a pool read on an admin route. + · `odoo_status` (customer) — WAS bare for the same reason and is NOT any more: lane E + answered `ASK D-15` by declaring `options: ['Active','Archived']` on the CONTRACT, which + is the right home (the grid picks a declared list up for free) and is why this function + never needed an arm for it. Recorded because the ask, not a guess, is what settled it. + """ + try: + import modules.inventory as inventory + labels = list(inventory.COVERAGE_LABELS or ()) + except Exception: + # A missing analytics import must not take the permission editor down; the dropdown + # degrades to today's empty one rather than the page to a 500. + return {} + return {"stock_bucket": labels} if labels else {} + + +def _clean_perms(v, governed_keys=None, session=None): + """Validate a whole `perms` block. FAIL-CLOSED, and LOUD rather than lenient. + + ⭐ WAVE 33 (W33-T33) — `governed_keys` IS PER TENANT AND IT IS NOT OPTIONAL IN PRACTICE. It + used to be `set(_PERM_MODULES)`, a literal, which is the same tenant-blindness owner item 11 + flagged: an admin of a tenant provisioned for neither grid module could still store a wall + for both. It now comes from `_perm_modules(session)`, i.e. from the tenant's own catalogue. + The default is the topic catalogue itself so a test or a future caller cannot silently widen + it. + + ⭐⭐ W36-T22 / CONTRACT C2 — **THE `enforced`/`listed` SPLIT IS GONE, AND SO IS + `unenforced_module`.** This function took TWO key sets and refused anything in the gap + between them, with a message that ended *"Share the database instead, or arm `perm_scope` + over `ut_*` first."* W36-T21 armed it. There is no gap left: every database the editor lists + is a database whose wall `routes_tables` applies on every read, so a wall stored against one + is enforced by the same code that enforces `customer_data`'s. **Two sets collapse into one**, + which is the shape C2 asks for — not a second set that is always equal to the first. + + ⚠ `session` IS THREADED SO `_module_fields` CAN READ A `ut_*` SCHEMA. Without it the + validator would have no field contract for the databases this ticket just made governable, + and "no contract" resolves to "every `hiddenFields` entry is unknown" — a 400 on the admin's + own screen. + + ⛔ WHY THIS REFUSES WHERE `clean_filter_tree` DROPS. The view sanitiser is deliberately + drop-per-node: one bad rule must never cost a user their whole saved view. A PERMISSION + filter is the opposite situation — an admin types "agent is Tara", a leaf is silently + dropped, and the stored wall is EMPTY. The admin sees "Saved", the account sees the whole + book, and nothing anywhere says so. So anything that would be dropped is a 400 instead: + the filter is re-validated with `clean_filter_tree` and the result must come back with the + SAME leaf count it went in with. + + Shape (C-PERM amendment 2 — a `FilterTree` is a PAIR, `{conj?, nodes}`; `clean_filter_tree` + validates the NODE LIST alone and never sees the root conjunction, so the two are checked + separately and a bare node list is refused rather than silently read as `and`). + """ + import aios_grid + + if v is None: + return None + if not isinstance(v, dict): + raise err(400, "bad_perms", "perms must be an object keyed by module") + governed = set(governed_keys) if governed_keys is not None else set(_PERM_MODULES) + unknown = sorted(set(v) - governed) + if unknown: + raise err(400, "bad_perms", + f"unknown module keys: {unknown} — this tenant's permission editor governs " + f"{sorted(governed)}") + out = {} + for key, raw in v.items(): + if not isinstance(raw, dict): + raise err(400, "bad_perms", f"{key}: each entry must be an object") + valid_keys = {f["key"] for f in _module_fields(key, session=session)} + hidden = raw.get("hiddenFields") or [] + if not isinstance(hidden, list): + raise err(400, "bad_perms", f"{key}: hiddenFields must be a list") + bad = sorted({str(h) for h in hidden} - valid_keys) + if bad: + # A hiddenFields entry naming nothing hides nothing — and reads as a restriction + # that is not there. + raise err(400, "bad_perms", + f"{key}: hiddenFields names unknown fields {bad}") + tree = raw.get("filter") + clean_tree = None + if tree not in (None, {}, []): + if not isinstance(tree, dict) or not isinstance(tree.get("nodes"), list): + raise err(400, "bad_filter", + f"{key}: filter must be {{conj?, nodes:[…]}} — a bare list would lose " + f"the root conjunction, and an 'or' wall read as 'and' restricts " + f"nothing it was meant to") + conj = tree.get("conj", "and") + if conj not in ("and", "or"): + raise err(400, "bad_filter", f"{key}: conj must be 'and' or 'or'") + nodes = tree["nodes"] + cleaned = aios_grid.clean_filter_tree(nodes, valid_keys, cohort_ids=None) + if _leaf_count(cleaned) != _leaf_count(nodes): + raise err(400, "bad_filter", + f"{key}: the filter contains conditions this module cannot evaluate " + f"(an unknown field, an unknown operator, or a cohort leaf — cohorts " + f"are per-user and cannot be a permanent rule). Refused rather than " + f"saved with the bad conditions silently removed, which would store a " + f"weaker wall than the one on screen.") + clean_tree = {"conj": conj, "nodes": cleaned} + # ⭐⭐ W38-T19 — `metrics` IS THE FOURTH FIELD OF AN ENTRY, AND ITS DEFAULT IS GRANT. + # `raw.get("metrics", True)` rather than a required key: a PUT composed by an older + # client, or a copy of a record written before this ticket, must not read as a + # revocation. `perm_scope.may_metrics` applies the identical rule on the read side, so + # the validator and the wall cannot disagree about what an absent key means. + out[key] = {"access": bool(raw.get("access", True)), + "filter": clean_tree, + "hiddenFields": sorted({str(h) for h in hidden}), + "metrics": bool(raw.get("metrics", True))} + _refuse_unshapeable_bu(out) + return out + + +def _refuse_unshapeable_bu(perms_out): + """⛔ A BU CONDITION THE PUSHDOWN CANNOT READ IS A VALUE LEAK WEARING A CORRECT ROW LIST. + + Amendment 3 in one more place. `perm_scope.derive_pool_scope` recognises exactly the shapes + that PIN a business unit: a top-level `dba eq` leaf, or a top-level OR-group of them — which + is what `perm_migrate` emits and what the condition builder produces for "is any of". An + admin composing the same intent a slightly different way (a NESTED group, `dba neq Royal`, a + `dba` leaf one level down) produces a filter that still narrows the ROWS correctly through + `permits()` — and leaves the pool built CONSOLIDATED, so every surviving row carries + Fisch+Royal numbers. The row list looks right. The revenue is another BU's. + + That is precisely the defect amendment 3 exists for, re-entering through the editor R9 just + shipped, and it cannot be caught downstream: by then the numbers are simply wrong, with + nothing anomalous about them. + + So it is refused at the WRITE, where a person is present to fix it. The discriminator is + narrow on purpose — the filter must MENTION `dba` and must fail to pin a team. A wall like + `dba is Fisch OR revenue > 1000` genuinely does not confine anyone to one BU, and + consolidated values are the correct answer for it, so it is not caught here (`derive` also + refuses it) and must not be. + """ + import core.perm_scope as perm_scope + + for key, e in perms_out.items(): + tree = e.get("filter") + if not tree or not _mentions_dba(tree.get("nodes")): + continue + probe = {"role": "user", "perms_v": perm_scope.PERMS_VERSION, + "perms": {key: {"access": True, "filter": tree, "hiddenFields": []}}} + team_id, _ = perm_scope.derive_pool_scope(probe, key) + if team_id is None and _confines_to_one_bu(tree): + raise err(400, "bu_condition_shape", + f"{key}: this filter restricts the business unit in a way the data layer " + f"cannot push into the query, so the rows would be correct while the " + f"revenue figures on them stayed consolidated across both units. Express " + f"the business-unit condition as a TOP-LEVEL condition — 'DBA is Fisch', " + f"or an 'any of' over the brands — and keep any other rules alongside it.") + + +def _guard_bu_widening(rec, cleaned, confirmed): + """⛔ THE TWO-CLICK LEAK: a SECOND save that silently drops the business-unit condition. + + `_fold_legacy_scope` runs only while a record is un-migrated, which is correct — after the + first save the filter IS the whole wall and the admin owns it. But the first save also + CHANGES the tree (it folds the BU condition in), so a client holding the tree it submitted + rather than the one the server returned will, on its next save, PUT a payload with no BU + condition — and a whole-record replace deletes it. Measured, not theorised: the same client + payload saved twice takes the pool scope from `(6, 'Ann')` to `(None, 'Ann')`, which is both + business units. + + So a save that REMOVES the brand confinement is refused unless it says it means to. Widening + BU access is a legitimate thing for an administrator to do — it is just never a thing to do + by accident, and the difference between the two is one explicit flag. + + ⚠ Deliberately NOT solved by always folding the legacy scope: that would make a BU + restriction permanent and unremovable, which contradicts R1 (the filter is the wall, and the + admin edits it). The right shape is "you may widen, say so". + """ + import core.perm_scope as perm_scope + + if confirmed: + return + stored = (rec.get("perms") or {}) if isinstance(rec.get("perms"), dict) else {} + for key, new_entry in cleaned.items(): + if not new_entry.get("access", True): + # Revoking the module entirely is the OPPOSITE of widening, and the filter on a + # denied entry governs nothing. Guarding it would make "lock this account out" + # require a confirm_widen flag, which reads as nonsense to whoever hits it. + continue + old = stored.get(key) + if not isinstance(old, dict): + continue # nothing stored yet: the fold already handled it + old_tree, new_tree = old.get("filter"), new_entry.get("filter") + if not old_tree or not _confines_to_one_bu(old_tree): + continue # was not confined -> this save cannot widen it + if new_tree and _confines_to_one_bu(new_tree): + continue # still confined -> not a widening + raise err(400, "bu_widening", + f"{key}: this would REMOVE the business-unit restriction and give the account " + f"both units. If that is intended, resend with confirm_widen: true. If it is " + f"not, reload the account's permissions first — saving a filter loaded before " + f"the last change drops the conditions it did not know about.") + # A guard is only as good as its trigger: `perm_scope` is imported so a future reader sees + # the connection to derive_pool_scope, which is what the removed condition was feeding. + _ = perm_scope + + +def _mentions_dba(nodes): + for n in nodes or (): + if not isinstance(n, dict): + continue + if isinstance(n.get("children"), list): + if _mentions_dba(n["children"]): + return True + elif n.get("colId") == "dba": + return True + return False + + +def _confines_to_one_bu(tree): + """Does the BRAND STRUCTURE ALONE confine the reader to one business unit? + + ⚠ "Alone" is the whole precision of this function, and getting it wrong makes the guard + refuse legitimate walls. `dba is Fisch OR ar_open > 1000` does NOT confine anybody — a + high-balance Royal customer satisfies it — so consolidated values are the correct answer and + refusing it would be a false positive. My first attempt evaluated synthetic rows carrying + only `dba`, which made `ar_open > 1000` read false on every probe row and turned that exact + wall into a "confinement" it is not. + + So every NON-`dba` leaf is treated as TRUE — the most generous reading, i.e. "suppose the + reader satisfies everything else; can they still see both brands?" If yes, the wall does not + confine and the pool is correctly consolidated. If no, the brand structure is doing the + confining and `derive_pool_scope` must be able to see it, or the numbers are wrong. + """ + def admits(node, brand): + if isinstance(node.get("children"), list): + kids = [admits(k, brand) for k in node["children"] if isinstance(k, dict)] + if not kids: + return True + return any(kids) if node.get("conj") == "or" else all(kids) + if node.get("colId") != "dba": + return True # every other condition: assume satisfied + want = str(node.get("value") or "").strip().lower() + op = node.get("op") + if op == "eq": + return brand.lower() == want + if op == "neq": + return brand.lower() != want + if op == "contains": + return want in brand.lower() + if op == "doesNotContain": + return want not in brand.lower() + if op == "isEmpty": + return False # a probe brand is never blank + if op == "isNotEmpty": + return True + return True # an op that cannot judge a brand does not confine + + nodes, conj = (tree.get("nodes") or []), tree.get("conj", "and") + root = {"conj": conj, "children": nodes} + admitted = {b for b in ("Fisch", "Royal", "Both") if admits(root, b)} + if not admitted: + return False # admits nothing at all — a different problem + # 'Both' belongs to either unit, so it cannot by itself widen the answer. + return not ({"Fisch", "Royal"} <= admitted) + + +def _leaf_count(nodes): + n = 0 + for node in nodes or (): + if isinstance(node, dict) and isinstance(node.get("children"), list): + n += _leaf_count(node["children"]) + else: + n += 1 + return n + + +def _fold_legacy_scope(rec, cleaned): + """⛔ MIGRATING A RECORD MUST NEVER WIDEN IT. The invariant this function exists to hold. + + Writing perms stamps `perms_v: 1`, and that marker SWITCHES OFF the legacy `bus`/`agent` + fallback in `perm_scope` (amendment 4 — absence must start meaning DENY). So a first write + whose filter carries no BU condition silently promotes a Royal-only account to BOTH business + units: the wall that used to come from `bus` is gone and nothing replaced it. + + Caught by the end-to-end leg in `verify_api` and by nothing else — every model-level check + passed, because the model was doing exactly what it was told. The account simply received + the other BU's customers on the wire. + + So on the FIRST perms write we fold the legacy scope in, as ordinary conditions, exactly as + `perm_migrate` would: this IS R1's migration, performed at the moment the record acquires a + perms block. From then on the admin sees those leaves in the editor (GET returns the folded + filter) and can change them like any other condition — which is R1's whole point, and why + this is a one-time fold rather than a permanent floor AND-ed on every write. + + Idempotent by construction: it runs only while the record is un-migrated, and the write that + calls it is the write that migrates it. + + ⛔ THE FOLD IS PER MODULE, BECAUSE THE LEGACY FILTER IS CUSTOMER-SHAPED AND THE MODULES ARE + NOT. `perm_migrate.filter_for` speaks `dba` and `agent` — both CUSTOMER columns. `product_data` + has neither (19 fields, no brand column at all), and `apply_row_scope` evaluates the permanent + filter with `permits()`, which DENIES anything unanswerable. So folding the customer wall into + the product entry does not narrow the product grid, it EMPTIES it: measured at 0 rows of 2 for + a `bus:[5]` account the moment an admin pressed Save. + + Nothing was wrong upstream — `_clean_perms` already refuses a `dba` condition typed against + `product_data` (its leaf keys are validated per topic). This fold was the ONLY door a + cross-topic leaf could come through, which is why it is the only place that needs the rule. + + ⚠ WHAT IS LOST HERE IS RECOVERED IN `perm_scope.derive_pool_scope`, NOT DISCARDED. Dropping the + BU condition from an AND root WIDENS that module's row scope, and for the product grid the BU + is a VALUES question anyway (`team_id` shapes `rev_ytd`/`qty_ytd`/`inv_value` — amendment 3). + The pushdown falls back to the record's own `bus` for exactly the modules whose field + vocabulary cannot express a BU, so the product pool is still BUILT as Fisch. The two halves + ship together or the second one is a leak. + """ + import core.perm_migrate as perm_migrate + import core.perm_scope as perm_scope + + if perm_scope.is_migrated(rec): + return cleaned + legacy = perm_migrate.filter_for(rec) + if not legacy: + return cleaned # nothing to preserve: the account was unrestricted + out = {} + for key, e in cleaned.items(): + legacy_here = _prune_to_module(legacy, {f["key"] for f in _module_fields(key)}) + tree = e.get("filter") + if not legacy_here: + # This module cannot evaluate ANY of the legacy conditions. Fold nothing rather than + # storing a wall it will read as "deny every row". + out[key] = e + continue + if not tree or not tree.get("nodes"): + folded = legacy_here + else: + # AND the two roots together. The submitted tree keeps its own conjunction by being + # nested as a GROUP — flattening an `or` tree into an `and` root would turn the + # admin's "A or B" into "A and B", which is a different and much narrower wall. + folded = {"conj": "and", + "nodes": list(legacy_here["nodes"]) + [{"conj": tree.get("conj", "and"), + "children": list(tree["nodes"])}]} + out[key] = dict(e, filter=folded) + return out + + +def _prune_to_module(tree, valid_keys): + """`tree` with every leaf this module cannot evaluate removed, or None if nothing survives. + + Used ONLY on the legacy fold above, where the alternative to pruning is a filter that denies + every row. It is not a general sanitiser: `_clean_perms` REFUSES rather than drops, for the + reason its own docstring gives (a silently-weakened wall reads as "Saved"). + + ⚠ A GROUP THAT LOSES ANY CHILD IS DROPPED WHOLE. Half of an `or` group is a narrower rule than + the admin's intent and half of an `and` group is a wider one; neither is the thing that was + written, and a fold has no standing to invent a third meaning. All-or-nothing per group keeps + the surviving conditions ones somebody actually declared. + """ + if not isinstance(tree, dict): + return None + + def keep(node): + if not isinstance(node, dict): + return None + kids = node.get("children") + if isinstance(kids, list): + surviving = [keep(k) for k in kids] + if not kids or any(s is None for s in surviving): + return None + return dict(node, children=surviving) + return node if node.get("colId") in valid_keys else None + + nodes = [n for n in (keep(n) for n in tree.get("nodes") or ()) if n is not None] + return {"conj": tree.get("conj", "and"), "nodes": nodes} if nodes else None + + +@router.get("/admin/users/{username}/perms") +def get_perms(username: str, session: Session = Depends(admin_gate)): + """This account's permission block plus the field schema the editor needs to render it.""" + reg = _registry() + uname = str(username or "").strip().lower() + if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): + # Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one — + # 404, never 403, because "that account exists in another company" is itself a leak. + raise err(404, "no_such_user", f"no account named {uname!r}") + import core.perm_scope as perm_scope + + rec = reg[uname] + stored = rec.get("perms") or {} + # S3 ask 3 — AN ENTRY FOR EVERY DECLARED MODULE, never a gap the client has to default. + # The PUT is a whole-record replace over this same module list, so a key declared here and + # omitted from `perms` would become an explicit DENY the moment anyone pressed Save. The + # default sent is the EFFECTIVE one (`may_access`), so an un-migrated record shows the + # access its legacy grant currently gives rather than a guess in either direction. + # + # ⭐⭐ WAVE 33 (W33-T33, owner item 11) — THE LIST IS THIS TENANT'S, NOT A LITERAL. Everything + # below was built from `_PERM_MODULES` and never touched `session.runtime`, which is why a + # nurilab or gtmlab admin opened Manage user and saw Royal Imports' Customer and Product + # databases. `_perm_modules` applies the tenant's own provisioning (the rule `routes_nav` + # has applied since wave 18) and merges the tenant's own `ut_*` databases. + modules = _perm_modules(session, surfaces=True) + # ⭐⭐ W36-T22 / C2 — EVERY listed database, not an `enforced` subset of them. See + # `perms.tenant_governable_modules` for why the flag is deleted rather than defaulted. + governed = [m["key"] for m in modules] + perms_out = {} + for k in governed: + e = stored.get(k) + # ⛔⛔ W36-T22 — THE DEFAULT IS `may_read`, NOT `may_access`, AND THE DIFFERENCE IS AN + # OUTAGE. `may_access` reads migrated-and-undeclared as DENY. That is right for a registry + # topic and catastrophic for a `ut_*` key, because no `ut_*` entry was STORABLE before this + # wave — so every migrated account carries none, and this route would default every one of + # them to `access: false`. + # + # ⛔ AND IT WOULD NOT HAVE STAYED A DISPLAY BUG FOR LONG. C2 also deleted the filters that + # kept `ut_*` keys OUT of the PUT body, so the editor now sends every declared module: an + # administrator opening this page, changing one unrelated filter and pressing Save would + # write an EXPLICIT `{access: false}` for all ten keychain databases — at which point + # `may_read`'s deny-only overlay fires and revokes them for real. One click, silent, + # permanent. Defaulting from the SAME evaluator the read door uses is what makes the page + # show what the user can actually open, so a save of an untouched payload is a no-op. + # ⚠ `_public(uname, rec)`, NOT the raw record. A stored record is keyed BY username in + # `users.json` and does not carry one INSIDE it, so `may_read` would hand `may_open` a + # `None` viewer and get a fail-closed False — the very outage this line exists to prevent, + # arriving through the fix for it. The same trap cost a gate double an hour earlier today. + # ⛔⛔ A SURFACE KEY DEFAULTS FROM `nav_may_open`, NOT `may_read`, AND THE + # DIFFERENCE IS A MASS REVOCATION. `may_read` on a non-`ut_` key falls through to + # `may_access`, which reads migrated-and-undeclared as DENY — correct for a topic + # (every save writes topic entries) and false for a surface, because NO surface entry + # was storable before 2026-08-18, so every migrated account carries none. The editor + # would paint Assistant unchecked for an account that opens it every day, and the next + # save of ANY unrelated change would write the explicit deny for real. `nav_may_open` + # is the evaluator the nav and the route gate actually ask, legacy fallback included, + # so the box shows what the account can reach and an untouched save stays a no-op. + _default = (perm_scope.nav_may_open(users._public(uname, rec), k) + if any(m.get("surface") and m["key"] == k for m in modules) else + perm_scope.may_read(users._public(uname, rec), k, st=session.runtime)) + # ⭐⭐ W38-T19 — `metrics` RIDES EVERY ENTRY, BACKFILLED TO GRANTED ON A RECORD WRITTEN + # BEFORE THE KEY EXISTED. The wire is TOTAL on purpose: the client's own parse already + # reads an absent key as granted, so this line changes no rendering — what it changes is + # what a HUMAN reads off the payload while debugging, and what the PUT round-trips. The + # stored dict is copied rather than mutated: `rec` is the live registry record and + # stamping a key onto it here would write a permission nobody saved. + perms_out[k] = (dict(e, metrics=bool(e.get("metrics", True))) + if isinstance(e, dict) else + {"access": bool(_default), "filter": None, "hiddenFields": [], + "metrics": True}) + return {"username": uname, + "perms": perms_out, + # S3 ask 2 — the marker rides the GET. An absent module entry means DENY on a + # migrated record and LEGACY on an un-migrated one: opposite meanings for identical + # JSON, so the client cannot render honestly without knowing which world it is in. + "perms_v": int(rec.get("perms_v") or 0), + # `role == 'admin'` bypasses perms entirely (amendment 4) — sent so the editor can + # say "Everything (admin)" instead of rendering stored rules that do not apply. + "is_admin": perms.is_admin(rec), + # ⭐⭐ W36-T22 / C2 — `{key, label}` per row and NO `enforced`. It used to ride here + # so the editor could paint an apology ("Not set here") for a database whose wall was + # never applied; W36-T21 armed every one of them, so the flag would now be constantly + # true and the apology would be a lie with a green gate behind it. The LABEL comes + # from the registry / the table's own definition. + "modules": modules, + # S3 ask 1 — CANONICAL SHAPE: a BARE ARRAY of fields per module key. Not the nav + # schema envelope; the editor needs the field list and nothing else, and one shape + # beats two readings of a sentence. + # ⭐ EVERY governed database now carries its fields, `ut_*` included — that is owner + # item 11's *"EVERY database should be able to be toggleable by admin"*, and it is + # the SAME resolver `_clean_perms` validates against, so the picker cannot offer a + # column the validator will reject. + "fields_by_module": {k: _module_fields(k, session=session) for k in governed}} + + +@router.put("/admin/users/{username}/perms") +def put_perms(username: str, body: dict = Body(default=None), + session: Session = Depends(admin_gate)): + """Replace this account's permission block wholesale. + + WHOLESALE, not a merge: "remove this restriction" has to be expressible, and a merge cannot + express a deletion without a second vocabulary for it. + + ⚠ NO EPOCH BUMP, for the same reason `PATCH /admin/users/{u}` does not bump one: + `deps._user_for` re-reads the record on every request, so a narrowed wall applies to the + target's LIVE session on its very next call. Bumping would only add a forced re-login on top + of a change that has already taken effect. + """ + body = body if isinstance(body, dict) else {} + reg = _registry() + uname = str(username or "").strip().lower() + if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): + # Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one — + # 404, never 403, because "that account exists in another company" is itself a leak. + raise err(404, "no_such_user", f"no account named {uname!r}") + if "perms" not in body: + raise err(400, "empty_patch", "no perms to save") + + # THE SELF-LOCKOUT GUARD, the twin of `self_demote` on the PATCH route. An admin bypasses + # perm_scope entirely, so this cannot brick them today — but it stops an administrator + # writing a wall for themselves that would bite the moment their role changed. + if uname == session.uname: + raise err(400, "self_perms", + "you cannot set permissions on your own account — ask another admin") + + _mods = _perm_modules(session, surfaces=True) + # ⭐⭐ W36-T22 / C2 — ONE key set. It was two ("listed" and "enforced") with a refusal in the + # gap; W36-T21 closed the gap, so every database the editor shows is one whose wall the table + # routes apply. `session` rides so a `ut_*` schema can be read for THIS tenant. + cleaned = _clean_perms(body.get("perms"), + governed_keys={m["key"] for m in _mods}, session=session) or {} + cleaned = _fold_legacy_scope(reg[uname], cleaned) + _guard_bu_widening(reg[uname], cleaned, bool(body.get("confirm_widen"))) + users.set_access(uname, perms=cleaned) + fresh = _registry() + rec = fresh.get(uname) or {} + if int(rec.get("perms_v") or 0) < users.PERMS_VERSION: + # The store took the write and did not record it. A 200 here would tell an administrator + # the wall is up when it is not — the one direction this must never fail in. + raise err(503, "store_unavailable", + "the permissions were not saved — the account is UNCHANGED") + return {"username": uname, "perms": rec.get("perms") or {}, + "perms_v": int(rec.get("perms_v") or 0)} + + +def _store_binding(session): + """`data_binding.describe()` for the store THIS SESSION's tenant actually writes. + + ⭐ THE TENANT'S STORE, NOT THE PROCESS DEFAULT, AND THAT DISTINCTION IS THE POINT. Tenant #0 + rides `core.store`'s module default, but nurilab/gtmlab/loopable are bound to their OWN dataset + repos by `harness/runtime.py:470`, straight out of the control-plane record — which is the path + that carried three tenants' PRODUCTION stores onto staging while `--data-repo` isolated only + tenant #0. Reporting the process default here would show "staging store, all fine" to exactly + the tenants that were not fine. + + ⚠ Degrades to a stated `unknown` rather than raising: a settings page that 500s because it + could not describe the store is worse than one that says it does not know. + """ + try: + import core.data_binding as data_binding # noqa: PLC0415 + import core.store as store # noqa: PLC0415 + bound = getattr(session.runtime, "store_handle", None) + repo = getattr(bound, "repo", None) or store.REPO + return data_binding.describe(repo) + except Exception as e: # noqa: BLE001 + return {"deployment": "unknown", "production": False, "repo": "", + "writable": False, "refusal": f"could not resolve the store binding: {e}", + "localOverride": False, "allowlist": [], "observed": {}} + + +@router.get("/settings") +def settings(session: Session = Depends(require_session)): + """The session's OWN settings — any authenticated user, not admin-only. + + `user` comes from `routes_auth._public_user`, deliberately reusing the ONE definition of what a + client may know about itself (never a hash, never the epoch). `scope` restates it as the two + things the Settings UI shows, and `tenant` names only THIS tenant — enumerating the others + would answer a question about our customer list. + """ + modules = perms.allowed_modules(session.user) + return { + # The build this Space is running. Behind the session on purpose — see `main.py::health`, + # which refuses it for being the first URL a scanner finds. `deploy_web.py` stamps it as a + # Space secret at every deploy; absent means "somebody started this container by hand". + "version": os.environ.get("AIOS_VERSION") or "unknown", + "user": _public_user(session.user), + "scope": { + "bus": perms.allowed_bu_labels(session.user), + "team_id": perms.scope_team_id(session.user), + "agent": perms.scope_agent(session.user), + "modules": sorted(modules) if modules is not None else "all", + # C-PERM: the caller's OWN effective wall, read-only. The grid uses it to grey + # pickers rather than to enforce — hidden fields are stripped from every data wire + # regardless, so a client that ignores this is narrowed anyway, never widened. + "perms": session.user.get("perms") or {}, + }, + "tenant": {"key": session.tenant, "name": getattr(session.runtime, "name", + session.tenant)}, + # ⭐⭐ D-315 — WHICH STORE THIS CONTAINER IS BOUND TO, AND WHETHER IT MAY WRITE IT. + # + # ⛔ THIS IS NOT DIAGNOSTIC GARNISH; IT IS THE ONLY WAY THE QUESTION CAN BE ANSWERED. + # D-160 established that a Space's environment cannot be read from outside, so "is staging + # pointed at production data?" is a question only the container can answer — and it went + # unanswered for the whole window in which two builds wrote tenant #0's real store. + # `verify_live` asserts this field after logging in, which is a check that survives a role + # swap in a way that reading a deploy log never did. + # + # ⚠ Session-gated, beside `version`, for `main.py::health`'s reason: a Space id and a + # dataset repo id are public names, but a health endpoint is the first URL a scanner finds + # and it may not describe the deployment. No customer data is here — only which store, and + # yes or no. + "store": _store_binding(session), + # CHROME ONLY. Every /admin route re-checks the role server-side; this exists so the client + # does not paint a "Manage users" button that 403s. + "admin": perms.is_admin(session.user), + # Wave 19 (R3, contract C2): the same courtesy for the LOOPABLE plane — the Settings rail + # paints its entry iff this is true, and `routes_platform_admin` refuses every request + # that is not, so a client that ignored this would gain exactly nothing. Derived from the + # ONE predicate rather than restated: a second copy of a two-condition wall is a second + # place for one of the conditions to go missing. + "platformAdmin": platform_admin.is_platform_admin(session.user), + } diff --git a/api/routes_agent_harness.py b/api/routes_agent_harness.py index ed34831c623af46ef28bd312848275948108c70b..cfe1ffa6958ea00035eaaaf64d5c826e246fb209 100644 --- a/api/routes_agent_harness.py +++ b/api/routes_agent_harness.py @@ -1,455 +1,455 @@ -"""routes_agent_harness.py — CONTRACT C4: the agent's HARNESS, kept as versioned files. - -Owner item 3, verbatim (2026-08-18): *"This new module under agent is supposed to host any file -pertaining to the agent skills, router etc. So we build a user can build a custom harness for each -agent through the chat interface."* - - GET /api/v1/agents/{id}/harness the file LIST (no bodies) - GET /api/v1/agents/{id}/harness?path=… one file: current body + its versions - GET /api/v1/agents/{id}/harness?path=…&version=N one older body, verbatim - PUT /api/v1/agents/{id}/harness write a NEW version of one file - DELETE /api/v1/agents/{id}/harness?path=… drop a file and its history - -⭐⭐ **R8 IS THE WHOLE SHAPE: BOTH PRINCIPALS WRITE, AND NOTHING IS EVER OVERWRITTEN.** A write -appends a version; a roll-back is a write of an older body (`restoredFrom`), never a delete. So the -history is a record of what happened rather than of what somebody last wanted it to look like. - -⛔ **R8 IS DELIBERATELY NOT R9.** An agent-authored ACTION's configuration is agent-only (item 8, -`routes_automation`); a harness file is not. Do not copy this file's posture over there or that -one's over here — the two rulings differ on purpose and they sit one screen apart in the product. - -⚠ **`author` IS THE SESSION, `authorKind` IS THE PROVENANCE, AND THEY ARE DIFFERENT FACTS.** Every -write through this router is made BY a signed-in administrator, so `author` is stamped from the -session and can never be supplied by the caller. `authorKind` says whether the BODY was drafted by -a person or by the agent in the chat — a claim the client is entitled to make, because both are -permitted (R8) and so nothing is bought by forging it. `record_version()` below is the server-side -door the automation engine uses when the agent writes with no session at all; that one stamps the -agent's own id as the author, which is the only case where `author` is not a username. - -⛔ **THE VERSION LIST IS CAPPED AND THE CAP IS REPORTED, NEVER SILENT.** `MAX_VERSIONS` versions of -one file are kept; past that the OLDEST are dropped and the count of what was dropped rides in -`trimmed` on every payload that mentions the file, so a reader can see that the history is partial -rather than infer that the file was only ever saved twice. This is the tenant document, which is -already 28.6 MB on tenant #0 and is deep-copied on every read: an unbounded per-agent history is -a store-sized leak with a UI in front of it. -""" -from datetime import datetime, timezone - -from fastapi import APIRouter, Body, Depends - -from deps import Session, err, require_session - -router = APIRouter(prefix="/api/v1") - -#: The tenant's harness files: `{agent_id: {path: file_record}}`. A per-tenant bucket, so it rides -#: `runtime.store_key`'s prefix and never lands in tenant #0's namespace — the same rule -#: `routes_slack.AGENTS_KEY` follows for the agent records these hang off. -HARNESS_KEY = "agent_harness" - -#: One body. Generous for a skill or a router file and far under the point where a single write -#: would move the tenant document measurably. A larger body is a REFUSAL at the door, not a -#: truncation: a silently truncated skill file is a harness that does not do what its text says. -MAX_BODY_BYTES = 128 * 1024 - -#: Files per agent. A refusal, not a trim — creating the 65th file is a different act from saving -#: the 101st version of one, and only the second can be a routine consequence of ordinary editing. -MAX_FILES = 64 - -#: Versions kept per file. Past this the oldest go and `trimmed` counts them (see the header). -MAX_VERSIONS = 100 - -MAX_PATH = 200 - -#: What a path may contain. ⛔ THIS IS NOT A FILESYSTEM PATH AND NOTHING HERE EVER TOUCHES A DISK — -#: the "files" are keys in a store bucket. The character rule exists so the key is displayable, is -#: safe to put in a URL, and cannot carry a traversal sequence that would look meaningful to a -#: future reader who assumes it IS a filesystem path. Fail-closed on the character set, not on a -#: list of forbidden sequences: an allow-list cannot be walked around by a spelling. -_PATH_OK = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-/") - -AUTHOR_KINDS = ("user", "agent") - - -def _now(): - return datetime.now(timezone.utc).isoformat(timespec="seconds") - - -def _all(rt): - """`{agent_id: {path: record}}` for one tenant. `{}` on any failure — an unreadable bucket must - degrade to "this agent has no harness files", never to a 500 on the pane that lists them.""" - try: - found = rt.get(HARNESS_KEY) or {} - except Exception: # noqa: BLE001 - return {} - return found if isinstance(found, dict) else {} - - -def _files(rt, agent_id): - found = _all(rt).get(str(agent_id)) - return found if isinstance(found, dict) else {} - - -#: What kind of principal an id names. `None` = this tenant has no agent with that id. -#: -#: ⛔⛔ THERE ARE TWO AGENT REGISTRIES IN THIS PRODUCT AND THE FIRST DRAFT OF THIS FILE KNEW ONLY -#: ONE (ASK D-5, 2026-08-18). `routes_slack._agents` is the per-Slack-channel permission wall under -#: Manage users. The **Agents module** the owner's item 3 is about is the AUTOMATION surface — -#: `Shell.tsx` mounts ``, and -#: `AutomationDetail`'s `panelTabs` is literally the `Properties | Run history` strip R7 adds -#: "Harness" to. Keyed on the Slack bucket alone, every `GET/PUT /agents/{id}/harness` from that -#: tab would have answered **404 no_agent**: whole, gate-green and dead on arrival -#: [[reachable-is-not-the-same-as-built]]. Verified independently before acting, not taken on -#: report: `surface="Agents"` mounts `AutomationSurface`, and `ManageAgentPane` contains ZERO -#: occurrences of Canvas, Properties, Run history or panelTabs. -#: -#: ⭐ ONE STORE, BOTH PRINCIPALS — never a second bucket keyed by surface. The owner's "agent -#: skills, router" files in two places is the parallel code path item 13 exists to refuse. -AGENT_SLACK = "slack" -AGENT_AUTOMATION = "automation" - - -def agent_kind(rt, agent_id): - """Which registry holds this id — `AGENT_SLACK`, `AGENT_AUTOMATION`, or `None`. - - ⚠ A SYNTHETIC ROW IS NOT AN AGENT FOR THIS PURPOSE. `field:` and `system:` ids are DERIVED at - read time from a column definition or a connector schedule; they have no stored home, so a - harness file hung off one is an orphan the moment the column changes. `patch_automation` - already refuses those ids for the neighbouring reason, and this refuses them by simply not - finding them — `all_definitions` holds stored automations only. - """ - import routes_slack - aid = str(agent_id or "") - if isinstance(routes_slack._agents(rt).get(aid), dict): - return AGENT_SLACK - import automation_engine as engine - try: - known = engine.all_definitions(rt) or {} - except Exception: # noqa: BLE001 - return None - return AGENT_AUTOMATION if aid in known else None - - -def _known_agent(rt, agent_id): - """Does this tenant have an agent with this id, in EITHER registry?""" - return agent_kind(rt, agent_id) is not None - - -def agent_wall(session, agent_id): - """404 for an unknown id, else apply the wall THAT PRINCIPAL'S OWN SURFACE applies. - - ⛔⛔ ONE DOOR, TWO WALLS, AND THAT IS NOT A SECOND CODE PATH — it is the refusal to invent a - THIRD wall. A Slack channel agent is administered under Manage users and every `/agents/*` door - in `routes_slack` is `admin_gate`; an automation lives in the Agents module and every door in - `routes_automation` is `module_gate("automation")`. A harness file is configuration OF the - agent it hangs off, so it is reached by whoever may already configure that agent. Picking one - of the two walls for both would either lock the Agents module's own users out of a tab the - owner asked for, or hand the Slack permission wall to anyone with an automation grant. - """ - kind = agent_kind(session.runtime, agent_id) - if kind is None: - raise err(404, "no_agent", "there is no agent with that id in this workspace") - if kind == AGENT_SLACK: - import core.perms as perms - if not perms.is_admin(session.user): - raise err(403, "forbidden", "administrators only") - else: - session.require("automation") - return kind - - -def normalize_path(raw): - """THE path rule, in ONE place. Returns the cleaned path, or `None` if it is not acceptable. - - ⛔⛔ ONE RULE, TWO DOORS, AND THAT IS WHY THIS IS A FUNCTION RATHER THAN TWO IF-BLOCKS. There - are two ways into this store — the HTTP route (which must answer 400) and `record_version()` - (which must raise `ValueError`, having no response to put a status into). Written twice, the - two copies are [[one-question-two-normalizers]] waiting to happen: the first weakening of one - copy is invisible because the other still refuses, so nothing goes red and the wall is now - half there. Written once, a change to the rule is felt at both doors and by the gate. - """ - path = str(raw or "").strip().strip("/") - if not path or len(path) > MAX_PATH: - return None - if set(path) - _PATH_OK or ".." in path or "//" in path: - return None - return path - - -def _clean_path(raw): - """`normalize_path` at the HTTP door, where a refusal is a 400 with a reason.""" - path = normalize_path(raw) - if path is None: - if not str(raw or "").strip().strip("/"): - raise err(400, "no_path", "a harness file needs a path, for example skills/router.md") - if len(str(raw)) > MAX_PATH: - raise err(400, "path_too_long", f"a harness path is at most {MAX_PATH} characters") - raise err(400, "bad_path", - "a harness path may use letters, digits, dot, dash, underscore and / only") - return path - - -def _blank(path, author, author_kind): - return {"path": path, "versions": [], "trimmed": 0, - "created": _now(), "createdBy": author, "createdKind": author_kind} - - -def _append(record, body, author, author_kind, restored_from=None): - """Append ONE version to a file record, in place, and report what was trimmed. - - The version NUMBER is monotonic and survives trimming — it counts writes, not stored entries. - A version list whose numbers restart at 1 after a trim would make two different bodies share a - name, and `restoredFrom` would then point at whichever one happened to be in the window. - """ - # ⚠ A NEW LIST, NEVER `versions.append(...)` ON THE STORED ONE. `update()` hands the callback - # the live document and may run it more than once; appending in place would then stack two - # copies of the same version into the history on a retry. - prior = record.get("versions") if isinstance(record.get("versions"), list) else [] - last = max((int(v.get("version") or 0) for v in prior if isinstance(v, dict)), default=0) - entry = {"version": last + 1, "body": body, "author": author, "authorKind": author_kind, - "created": _now(), "bytes": len(body.encode("utf-8"))} - if restored_from: - entry["restoredFrom"] = int(restored_from) - versions = [*prior, entry] - dropped = max(0, len(versions) - MAX_VERSIONS) - if dropped: - versions = versions[dropped:] - record["versions"] = versions - record["trimmed"] = int(record.get("trimmed") or 0) + dropped - return entry - - -def _head(record): - """The newest version of a file record, or `None` for a record with no versions at all.""" - versions = record.get("versions") if isinstance(record.get("versions"), list) else [] - return versions[-1] if versions else None - - -def _row(record): - """One file, as the LIST door reports it: everything except the bodies. - - ⚠ NO BODY, AND THAT IS THE POINT. A list door that carried every version of every file would - ship the whole harness on every pane render; the Harness tab lists first and opens one file - second, which is exactly the shape this answers. - """ - head = _head(record) or {} - versions = record.get("versions") if isinstance(record.get("versions"), list) else [] - return {"path": record.get("path") or "", - "version": int(head.get("version") or 0), - "bytes": int(head.get("bytes") or 0), - "author": head.get("author") or record.get("createdBy") or "", - "authorKind": head.get("authorKind") or record.get("createdKind") or "user", - "updated": head.get("created") or record.get("created") or "", - "created": record.get("created") or "", - "versions": len(versions), - "trimmed": int(record.get("trimmed") or 0)} - - -def _version_rows(record): - """The history of one file, newest first, WITHOUT the bodies. - - A body per version is what makes a diff possible, and it is also what makes this payload big: - 100 versions of a 128 KB file is 12 MB. The client asks for the two bodies it is diffing - (`?path=…&version=N`), which is two round trips for a diff and none for a history list. - """ - versions = record.get("versions") if isinstance(record.get("versions"), list) else [] - out = [] - for entry in reversed(versions): - if not isinstance(entry, dict): - continue - row = {"version": int(entry.get("version") or 0), - "author": entry.get("author") or "", - "authorKind": entry.get("authorKind") or "user", - "created": entry.get("created") or "", - "bytes": int(entry.get("bytes") or 0)} - if entry.get("restoredFrom"): - row["restoredFrom"] = int(entry["restoredFrom"]) - out.append(row) - return out - - -def _limits(): - """The caps, IN the payload, so a client can say "this file is full" before a write fails. - - ⚠ A limit the client cannot see is a limit the user meets as an error. `trimmed` reports the - one cap that acts without refusing; these report the three that refuse. - """ - return {"maxBodyBytes": MAX_BODY_BYTES, "maxFiles": MAX_FILES, - "maxVersions": MAX_VERSIONS, "maxPath": MAX_PATH} - - -# ── the write door the SERVER uses (no session) ──────────────────────────────────────────────── -def record_version(runtime, agent_id, path, body, author="", author_kind="agent", - restored_from=None): - """Write one version from INSIDE the server — the agent's own half of R8. - - ⭐ THIS IS THE FUNCTION, NOT THE ROUTE, THAT MAKES "editable by BOTH the user and the agent" - true. An agent acting inside an automation run holds no session and no cookie; if its only way - to write were the HTTP door it would have to borrow a person's identity, and the authorship - column would then be a record of who was logged in rather than of what wrote the file. - - Returns the appended entry. Raises nothing the caller cannot handle: an unknown agent is a - `ValueError`, because a server-side caller has no HTTP response to put a 404 into. - """ - agent_id = str(agent_id or "") - if not _known_agent(runtime, agent_id): - raise ValueError(f"no agent {agent_id!r} in this workspace") - path = normalize_path(path) - if path is None: - raise ValueError("that is not an acceptable harness path") - body = str(body or "") - if len(body.encode("utf-8")) > MAX_BODY_BYTES: - raise ValueError("harness body is over the size limit") - kind = author_kind if author_kind in AUTHOR_KINDS else "agent" - author = str(author or agent_id) - # ⛔ THE FILE-COUNT CEILING IS CHECKED HERE, NOT INSIDE `_set`. An exception raised inside a - # store `update` callback propagates out of a half-run read-modify-write, and the one thing a - # refusal must never do is leave the caller unsure whether the write happened. - existing = _files(runtime, agent_id) - if path not in existing and len(existing) >= MAX_FILES: - raise ValueError(f"this agent already has {MAX_FILES} harness files") - appended = {} - - def _set(cur): - cur = dict(cur or {}) - files = dict(cur.get(agent_id) or {}) if isinstance(cur.get(agent_id), dict) else {} - record = dict(files[path]) if isinstance(files.get(path), dict) else _blank(path, author, kind) - appended.clear() - appended.update(_append(record, body, author, kind, restored_from)) - files[path] = record - cur[agent_id] = files - return cur - - runtime.update(HARNESS_KEY, _set, flush="sync") - return appended - - -# ── the routes ──────────────────────────────────────────────────────────────────────────────── -@router.get("/agents/{agent_id}/harness") -def get_harness(agent_id: str, path: str = "", version: int = 0, - session: Session = Depends(require_session)): - """The list, one file, or one older body — decided by the query string (C4). - - ⚠ ADMIN-GATED LIKE EVERY OTHER AGENT DOOR (`routes_slack`), and for a stronger reason than - consistency: a harness file is what the agent is INSTRUCTED to do, so writing one is closer to - editing a permission than to editing a document. - """ - agent_id = str(agent_id or "") - agent_wall(session, agent_id) - files = _files(session.runtime, agent_id) - if not path: - rows = [_row(rec) for _p, rec in sorted(files.items()) if isinstance(rec, dict)] - return {"agent": agent_id, "files": rows, "limits": _limits()} - - wanted = _clean_path(path) - record = files.get(wanted) - if not isinstance(record, dict): - raise err(404, "no_file", f"this agent has no harness file at {wanted}") - versions = record.get("versions") if isinstance(record.get("versions"), list) else [] - if version: - for entry in versions: - if isinstance(entry, dict) and int(entry.get("version") or 0) == int(version): - return {"agent": agent_id, **_row(record), "body": entry.get("body") or "", - "atVersion": int(version), "history": _version_rows(record), - "limits": _limits()} - # ⛔ A TRIMMED VERSION IS A NAMED REFUSAL, NOT A 404 SHAPED LIKE A TYPO. The client asked - # for something that existed and no longer does, and telling it apart from a bad number is - # the difference between "roll back to v3" failing loudly and failing as if v3 never was. - trimmed = int(record.get("trimmed") or 0) - if trimmed and int(version) <= trimmed: - raise err(410, "version_trimmed", - f"version {int(version)} is older than the {MAX_VERSIONS} versions kept " - f"for this file, and its body is gone") - raise err(404, "no_version", f"this file has no version {int(version)}") - head = _head(record) or {} - return {"agent": agent_id, **_row(record), "body": head.get("body") or "", - "atVersion": int(head.get("version") or 0), "history": _version_rows(record), - "limits": _limits()} - - -@router.put("/agents/{agent_id}/harness") -def put_harness(agent_id: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Write a NEW version of one harness file. `{path, body, authorKind?, restoredFrom?}` (C4). - - ⛔ THERE IS NO OVERWRITE HERE AND THERE IS NO EDIT-IN-PLACE. R8's "every version kept" is not a - UI affordance; it is this function refusing to have a code path that replaces a body. A - roll-back arrives as an ordinary write carrying `restoredFrom`, so the history records that - somebody went back rather than pretending the intervening versions never happened. - """ - agent_id = str(agent_id or "") - agent_wall(session, agent_id) - body = body if isinstance(body, dict) else {} - path = _clean_path(body.get("path")) - text = body.get("body") - if not isinstance(text, str): - raise err(400, "no_body", "a harness file needs a body, even an empty one") - if len(text.encode("utf-8")) > MAX_BODY_BYTES: - raise err(413, "body_too_long", - f"a harness file is at most {MAX_BODY_BYTES // 1024} KB; this one is larger") - - # ⚠ THE CALLER DECLARES THE PROVENANCE AND THE SERVER STAMPS THE IDENTITY. `authorKind` is a - # claim about who WROTE the text (the person, or the agent in the chat panel); `author` is the - # session and is never read off the request. Nothing is bought by forging the first — both - # principals may write (R8) — and everything would be bought by forging the second. - kind = str(body.get("authorKind") or "user").strip().lower() - if kind not in AUTHOR_KINDS: - raise err(400, "bad_author_kind", "authorKind is either user or agent") - restored = body.get("restoredFrom") - try: - restored = int(restored) if restored else None - except (TypeError, ValueError): - raise err(400, "bad_version", "restoredFrom must be a version number") - - files = _files(session.runtime, agent_id) - if path not in files and len(files) >= MAX_FILES: - raise err(409, "too_many_files", - f"this agent already has {MAX_FILES} harness files; delete one to add another") - try: - record_version(session.runtime, agent_id, path, text, - author=session.uname, author_kind=kind, restored_from=restored) - except ValueError as exc: - raise err(400, "refused", str(exc)) - - fresh = _files(session.runtime, agent_id).get(path) - if not isinstance(fresh, dict) or not _head(fresh): - # The store took the write and did not record it. A 200 here would tell an administrator - # their skill file was saved when it was not — the shape `routes_slack` refuses too. - raise err(503, "store_unavailable", "the harness file was NOT saved") - head = _head(fresh) - return {"agent": agent_id, **_row(fresh), "body": head.get("body") or "", - "atVersion": int(head.get("version") or 0), "history": _version_rows(fresh), - "limits": _limits()} - - -@router.delete("/agents/{agent_id}/harness") -def delete_harness(agent_id: str, path: str = "", session: Session = Depends(require_session)): - """Drop one harness file AND its history. - - ⚠ THIS IS NOT THE THING R8 FORBIDS. R8 forbids a ROLL-BACK implemented as a delete — losing - versions as a side effect of an edit. Deleting a file is a person deciding the file should not - exist, which is a different act with a different button, and a store with no way to remove a - file is one where a typo'd path is permanent. - """ - agent_id = str(agent_id or "") - agent_wall(session, agent_id) - wanted = _clean_path(path) - if wanted not in _files(session.runtime, agent_id): - raise err(404, "no_file", f"this agent has no harness file at {wanted}") - - def _set(cur): - cur = dict(cur or {}) - files = dict(cur.get(agent_id) or {}) if isinstance(cur.get(agent_id), dict) else {} - files.pop(wanted, None) - # An agent with no harness files leaves NO key behind. An empty dict per agent id is how a - # bucket accumulates a row for every agent anybody ever opened the tab on. - if files: - cur[agent_id] = files - else: - cur.pop(agent_id, None) - return cur - - session.runtime.update(HARNESS_KEY, _set, flush="sync") - return {"agent": agent_id, "deleted": wanted, - "files": [_row(rec) for _p, rec in sorted(_files(session.runtime, agent_id).items()) - if isinstance(rec, dict)], - "limits": _limits()} +"""routes_agent_harness.py — CONTRACT C4: the agent's HARNESS, kept as versioned files. + +Owner item 3, verbatim (2026-08-18): *"This new module under agent is supposed to host any file +pertaining to the agent skills, router etc. So we build a user can build a custom harness for each +agent through the chat interface."* + + GET /api/v1/agents/{id}/harness the file LIST (no bodies) + GET /api/v1/agents/{id}/harness?path=… one file: current body + its versions + GET /api/v1/agents/{id}/harness?path=…&version=N one older body, verbatim + PUT /api/v1/agents/{id}/harness write a NEW version of one file + DELETE /api/v1/agents/{id}/harness?path=… drop a file and its history + +⭐⭐ **R8 IS THE WHOLE SHAPE: BOTH PRINCIPALS WRITE, AND NOTHING IS EVER OVERWRITTEN.** A write +appends a version; a roll-back is a write of an older body (`restoredFrom`), never a delete. So the +history is a record of what happened rather than of what somebody last wanted it to look like. + +⛔ **R8 IS DELIBERATELY NOT R9.** An agent-authored ACTION's configuration is agent-only (item 8, +`routes_automation`); a harness file is not. Do not copy this file's posture over there or that +one's over here — the two rulings differ on purpose and they sit one screen apart in the product. + +⚠ **`author` IS THE SESSION, `authorKind` IS THE PROVENANCE, AND THEY ARE DIFFERENT FACTS.** Every +write through this router is made BY a signed-in administrator, so `author` is stamped from the +session and can never be supplied by the caller. `authorKind` says whether the BODY was drafted by +a person or by the agent in the chat — a claim the client is entitled to make, because both are +permitted (R8) and so nothing is bought by forging it. `record_version()` below is the server-side +door the automation engine uses when the agent writes with no session at all; that one stamps the +agent's own id as the author, which is the only case where `author` is not a username. + +⛔ **THE VERSION LIST IS CAPPED AND THE CAP IS REPORTED, NEVER SILENT.** `MAX_VERSIONS` versions of +one file are kept; past that the OLDEST are dropped and the count of what was dropped rides in +`trimmed` on every payload that mentions the file, so a reader can see that the history is partial +rather than infer that the file was only ever saved twice. This is the tenant document, which is +already 28.6 MB on tenant #0 and is deep-copied on every read: an unbounded per-agent history is +a store-sized leak with a UI in front of it. +""" +from datetime import datetime, timezone + +from fastapi import APIRouter, Body, Depends + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + +#: The tenant's harness files: `{agent_id: {path: file_record}}`. A per-tenant bucket, so it rides +#: `runtime.store_key`'s prefix and never lands in tenant #0's namespace — the same rule +#: `routes_slack.AGENTS_KEY` follows for the agent records these hang off. +HARNESS_KEY = "agent_harness" + +#: One body. Generous for a skill or a router file and far under the point where a single write +#: would move the tenant document measurably. A larger body is a REFUSAL at the door, not a +#: truncation: a silently truncated skill file is a harness that does not do what its text says. +MAX_BODY_BYTES = 128 * 1024 + +#: Files per agent. A refusal, not a trim — creating the 65th file is a different act from saving +#: the 101st version of one, and only the second can be a routine consequence of ordinary editing. +MAX_FILES = 64 + +#: Versions kept per file. Past this the oldest go and `trimmed` counts them (see the header). +MAX_VERSIONS = 100 + +MAX_PATH = 200 + +#: What a path may contain. ⛔ THIS IS NOT A FILESYSTEM PATH AND NOTHING HERE EVER TOUCHES A DISK — +#: the "files" are keys in a store bucket. The character rule exists so the key is displayable, is +#: safe to put in a URL, and cannot carry a traversal sequence that would look meaningful to a +#: future reader who assumes it IS a filesystem path. Fail-closed on the character set, not on a +#: list of forbidden sequences: an allow-list cannot be walked around by a spelling. +_PATH_OK = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-/") + +AUTHOR_KINDS = ("user", "agent") + + +def _now(): + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +def _all(rt): + """`{agent_id: {path: record}}` for one tenant. `{}` on any failure — an unreadable bucket must + degrade to "this agent has no harness files", never to a 500 on the pane that lists them.""" + try: + found = rt.get(HARNESS_KEY) or {} + except Exception: # noqa: BLE001 + return {} + return found if isinstance(found, dict) else {} + + +def _files(rt, agent_id): + found = _all(rt).get(str(agent_id)) + return found if isinstance(found, dict) else {} + + +#: What kind of principal an id names. `None` = this tenant has no agent with that id. +#: +#: ⛔⛔ THERE ARE TWO AGENT REGISTRIES IN THIS PRODUCT AND THE FIRST DRAFT OF THIS FILE KNEW ONLY +#: ONE (ASK D-5, 2026-08-18). `routes_slack._agents` is the per-Slack-channel permission wall under +#: Manage users. The **Agents module** the owner's item 3 is about is the AUTOMATION surface — +#: `Shell.tsx` mounts ``, and +#: `AutomationDetail`'s `panelTabs` is literally the `Properties | Run history` strip R7 adds +#: "Harness" to. Keyed on the Slack bucket alone, every `GET/PUT /agents/{id}/harness` from that +#: tab would have answered **404 no_agent**: whole, gate-green and dead on arrival +#: [[reachable-is-not-the-same-as-built]]. Verified independently before acting, not taken on +#: report: `surface="Agents"` mounts `AutomationSurface`, and `ManageAgentPane` contains ZERO +#: occurrences of Canvas, Properties, Run history or panelTabs. +#: +#: ⭐ ONE STORE, BOTH PRINCIPALS — never a second bucket keyed by surface. The owner's "agent +#: skills, router" files in two places is the parallel code path item 13 exists to refuse. +AGENT_SLACK = "slack" +AGENT_AUTOMATION = "automation" + + +def agent_kind(rt, agent_id): + """Which registry holds this id — `AGENT_SLACK`, `AGENT_AUTOMATION`, or `None`. + + ⚠ A SYNTHETIC ROW IS NOT AN AGENT FOR THIS PURPOSE. `field:` and `system:` ids are DERIVED at + read time from a column definition or a connector schedule; they have no stored home, so a + harness file hung off one is an orphan the moment the column changes. `patch_automation` + already refuses those ids for the neighbouring reason, and this refuses them by simply not + finding them — `all_definitions` holds stored automations only. + """ + import routes_slack + aid = str(agent_id or "") + if isinstance(routes_slack._agents(rt).get(aid), dict): + return AGENT_SLACK + import automation_engine as engine + try: + known = engine.all_definitions(rt) or {} + except Exception: # noqa: BLE001 + return None + return AGENT_AUTOMATION if aid in known else None + + +def _known_agent(rt, agent_id): + """Does this tenant have an agent with this id, in EITHER registry?""" + return agent_kind(rt, agent_id) is not None + + +def agent_wall(session, agent_id): + """404 for an unknown id, else apply the wall THAT PRINCIPAL'S OWN SURFACE applies. + + ⛔⛔ ONE DOOR, TWO WALLS, AND THAT IS NOT A SECOND CODE PATH — it is the refusal to invent a + THIRD wall. A Slack channel agent is administered under Manage users and every `/agents/*` door + in `routes_slack` is `admin_gate`; an automation lives in the Agents module and every door in + `routes_automation` is `module_gate("automation")`. A harness file is configuration OF the + agent it hangs off, so it is reached by whoever may already configure that agent. Picking one + of the two walls for both would either lock the Agents module's own users out of a tab the + owner asked for, or hand the Slack permission wall to anyone with an automation grant. + """ + kind = agent_kind(session.runtime, agent_id) + if kind is None: + raise err(404, "no_agent", "there is no agent with that id in this workspace") + if kind == AGENT_SLACK: + import core.perms as perms + if not perms.is_admin(session.user): + raise err(403, "forbidden", "administrators only") + else: + session.require("automation") + return kind + + +def normalize_path(raw): + """THE path rule, in ONE place. Returns the cleaned path, or `None` if it is not acceptable. + + ⛔⛔ ONE RULE, TWO DOORS, AND THAT IS WHY THIS IS A FUNCTION RATHER THAN TWO IF-BLOCKS. There + are two ways into this store — the HTTP route (which must answer 400) and `record_version()` + (which must raise `ValueError`, having no response to put a status into). Written twice, the + two copies are [[one-question-two-normalizers]] waiting to happen: the first weakening of one + copy is invisible because the other still refuses, so nothing goes red and the wall is now + half there. Written once, a change to the rule is felt at both doors and by the gate. + """ + path = str(raw or "").strip().strip("/") + if not path or len(path) > MAX_PATH: + return None + if set(path) - _PATH_OK or ".." in path or "//" in path: + return None + return path + + +def _clean_path(raw): + """`normalize_path` at the HTTP door, where a refusal is a 400 with a reason.""" + path = normalize_path(raw) + if path is None: + if not str(raw or "").strip().strip("/"): + raise err(400, "no_path", "a harness file needs a path, for example skills/router.md") + if len(str(raw)) > MAX_PATH: + raise err(400, "path_too_long", f"a harness path is at most {MAX_PATH} characters") + raise err(400, "bad_path", + "a harness path may use letters, digits, dot, dash, underscore and / only") + return path + + +def _blank(path, author, author_kind): + return {"path": path, "versions": [], "trimmed": 0, + "created": _now(), "createdBy": author, "createdKind": author_kind} + + +def _append(record, body, author, author_kind, restored_from=None): + """Append ONE version to a file record, in place, and report what was trimmed. + + The version NUMBER is monotonic and survives trimming — it counts writes, not stored entries. + A version list whose numbers restart at 1 after a trim would make two different bodies share a + name, and `restoredFrom` would then point at whichever one happened to be in the window. + """ + # ⚠ A NEW LIST, NEVER `versions.append(...)` ON THE STORED ONE. `update()` hands the callback + # the live document and may run it more than once; appending in place would then stack two + # copies of the same version into the history on a retry. + prior = record.get("versions") if isinstance(record.get("versions"), list) else [] + last = max((int(v.get("version") or 0) for v in prior if isinstance(v, dict)), default=0) + entry = {"version": last + 1, "body": body, "author": author, "authorKind": author_kind, + "created": _now(), "bytes": len(body.encode("utf-8"))} + if restored_from: + entry["restoredFrom"] = int(restored_from) + versions = [*prior, entry] + dropped = max(0, len(versions) - MAX_VERSIONS) + if dropped: + versions = versions[dropped:] + record["versions"] = versions + record["trimmed"] = int(record.get("trimmed") or 0) + dropped + return entry + + +def _head(record): + """The newest version of a file record, or `None` for a record with no versions at all.""" + versions = record.get("versions") if isinstance(record.get("versions"), list) else [] + return versions[-1] if versions else None + + +def _row(record): + """One file, as the LIST door reports it: everything except the bodies. + + ⚠ NO BODY, AND THAT IS THE POINT. A list door that carried every version of every file would + ship the whole harness on every pane render; the Harness tab lists first and opens one file + second, which is exactly the shape this answers. + """ + head = _head(record) or {} + versions = record.get("versions") if isinstance(record.get("versions"), list) else [] + return {"path": record.get("path") or "", + "version": int(head.get("version") or 0), + "bytes": int(head.get("bytes") or 0), + "author": head.get("author") or record.get("createdBy") or "", + "authorKind": head.get("authorKind") or record.get("createdKind") or "user", + "updated": head.get("created") or record.get("created") or "", + "created": record.get("created") or "", + "versions": len(versions), + "trimmed": int(record.get("trimmed") or 0)} + + +def _version_rows(record): + """The history of one file, newest first, WITHOUT the bodies. + + A body per version is what makes a diff possible, and it is also what makes this payload big: + 100 versions of a 128 KB file is 12 MB. The client asks for the two bodies it is diffing + (`?path=…&version=N`), which is two round trips for a diff and none for a history list. + """ + versions = record.get("versions") if isinstance(record.get("versions"), list) else [] + out = [] + for entry in reversed(versions): + if not isinstance(entry, dict): + continue + row = {"version": int(entry.get("version") or 0), + "author": entry.get("author") or "", + "authorKind": entry.get("authorKind") or "user", + "created": entry.get("created") or "", + "bytes": int(entry.get("bytes") or 0)} + if entry.get("restoredFrom"): + row["restoredFrom"] = int(entry["restoredFrom"]) + out.append(row) + return out + + +def _limits(): + """The caps, IN the payload, so a client can say "this file is full" before a write fails. + + ⚠ A limit the client cannot see is a limit the user meets as an error. `trimmed` reports the + one cap that acts without refusing; these report the three that refuse. + """ + return {"maxBodyBytes": MAX_BODY_BYTES, "maxFiles": MAX_FILES, + "maxVersions": MAX_VERSIONS, "maxPath": MAX_PATH} + + +# ── the write door the SERVER uses (no session) ──────────────────────────────────────────────── +def record_version(runtime, agent_id, path, body, author="", author_kind="agent", + restored_from=None): + """Write one version from INSIDE the server — the agent's own half of R8. + + ⭐ THIS IS THE FUNCTION, NOT THE ROUTE, THAT MAKES "editable by BOTH the user and the agent" + true. An agent acting inside an automation run holds no session and no cookie; if its only way + to write were the HTTP door it would have to borrow a person's identity, and the authorship + column would then be a record of who was logged in rather than of what wrote the file. + + Returns the appended entry. Raises nothing the caller cannot handle: an unknown agent is a + `ValueError`, because a server-side caller has no HTTP response to put a 404 into. + """ + agent_id = str(agent_id or "") + if not _known_agent(runtime, agent_id): + raise ValueError(f"no agent {agent_id!r} in this workspace") + path = normalize_path(path) + if path is None: + raise ValueError("that is not an acceptable harness path") + body = str(body or "") + if len(body.encode("utf-8")) > MAX_BODY_BYTES: + raise ValueError("harness body is over the size limit") + kind = author_kind if author_kind in AUTHOR_KINDS else "agent" + author = str(author or agent_id) + # ⛔ THE FILE-COUNT CEILING IS CHECKED HERE, NOT INSIDE `_set`. An exception raised inside a + # store `update` callback propagates out of a half-run read-modify-write, and the one thing a + # refusal must never do is leave the caller unsure whether the write happened. + existing = _files(runtime, agent_id) + if path not in existing and len(existing) >= MAX_FILES: + raise ValueError(f"this agent already has {MAX_FILES} harness files") + appended = {} + + def _set(cur): + cur = dict(cur or {}) + files = dict(cur.get(agent_id) or {}) if isinstance(cur.get(agent_id), dict) else {} + record = dict(files[path]) if isinstance(files.get(path), dict) else _blank(path, author, kind) + appended.clear() + appended.update(_append(record, body, author, kind, restored_from)) + files[path] = record + cur[agent_id] = files + return cur + + runtime.update(HARNESS_KEY, _set, flush="sync") + return appended + + +# ── the routes ──────────────────────────────────────────────────────────────────────────────── +@router.get("/agents/{agent_id}/harness") +def get_harness(agent_id: str, path: str = "", version: int = 0, + session: Session = Depends(require_session)): + """The list, one file, or one older body — decided by the query string (C4). + + ⚠ ADMIN-GATED LIKE EVERY OTHER AGENT DOOR (`routes_slack`), and for a stronger reason than + consistency: a harness file is what the agent is INSTRUCTED to do, so writing one is closer to + editing a permission than to editing a document. + """ + agent_id = str(agent_id or "") + agent_wall(session, agent_id) + files = _files(session.runtime, agent_id) + if not path: + rows = [_row(rec) for _p, rec in sorted(files.items()) if isinstance(rec, dict)] + return {"agent": agent_id, "files": rows, "limits": _limits()} + + wanted = _clean_path(path) + record = files.get(wanted) + if not isinstance(record, dict): + raise err(404, "no_file", f"this agent has no harness file at {wanted}") + versions = record.get("versions") if isinstance(record.get("versions"), list) else [] + if version: + for entry in versions: + if isinstance(entry, dict) and int(entry.get("version") or 0) == int(version): + return {"agent": agent_id, **_row(record), "body": entry.get("body") or "", + "atVersion": int(version), "history": _version_rows(record), + "limits": _limits()} + # ⛔ A TRIMMED VERSION IS A NAMED REFUSAL, NOT A 404 SHAPED LIKE A TYPO. The client asked + # for something that existed and no longer does, and telling it apart from a bad number is + # the difference between "roll back to v3" failing loudly and failing as if v3 never was. + trimmed = int(record.get("trimmed") or 0) + if trimmed and int(version) <= trimmed: + raise err(410, "version_trimmed", + f"version {int(version)} is older than the {MAX_VERSIONS} versions kept " + f"for this file, and its body is gone") + raise err(404, "no_version", f"this file has no version {int(version)}") + head = _head(record) or {} + return {"agent": agent_id, **_row(record), "body": head.get("body") or "", + "atVersion": int(head.get("version") or 0), "history": _version_rows(record), + "limits": _limits()} + + +@router.put("/agents/{agent_id}/harness") +def put_harness(agent_id: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Write a NEW version of one harness file. `{path, body, authorKind?, restoredFrom?}` (C4). + + ⛔ THERE IS NO OVERWRITE HERE AND THERE IS NO EDIT-IN-PLACE. R8's "every version kept" is not a + UI affordance; it is this function refusing to have a code path that replaces a body. A + roll-back arrives as an ordinary write carrying `restoredFrom`, so the history records that + somebody went back rather than pretending the intervening versions never happened. + """ + agent_id = str(agent_id or "") + agent_wall(session, agent_id) + body = body if isinstance(body, dict) else {} + path = _clean_path(body.get("path")) + text = body.get("body") + if not isinstance(text, str): + raise err(400, "no_body", "a harness file needs a body, even an empty one") + if len(text.encode("utf-8")) > MAX_BODY_BYTES: + raise err(413, "body_too_long", + f"a harness file is at most {MAX_BODY_BYTES // 1024} KB; this one is larger") + + # ⚠ THE CALLER DECLARES THE PROVENANCE AND THE SERVER STAMPS THE IDENTITY. `authorKind` is a + # claim about who WROTE the text (the person, or the agent in the chat panel); `author` is the + # session and is never read off the request. Nothing is bought by forging the first — both + # principals may write (R8) — and everything would be bought by forging the second. + kind = str(body.get("authorKind") or "user").strip().lower() + if kind not in AUTHOR_KINDS: + raise err(400, "bad_author_kind", "authorKind is either user or agent") + restored = body.get("restoredFrom") + try: + restored = int(restored) if restored else None + except (TypeError, ValueError): + raise err(400, "bad_version", "restoredFrom must be a version number") + + files = _files(session.runtime, agent_id) + if path not in files and len(files) >= MAX_FILES: + raise err(409, "too_many_files", + f"this agent already has {MAX_FILES} harness files; delete one to add another") + try: + record_version(session.runtime, agent_id, path, text, + author=session.uname, author_kind=kind, restored_from=restored) + except ValueError as exc: + raise err(400, "refused", str(exc)) + + fresh = _files(session.runtime, agent_id).get(path) + if not isinstance(fresh, dict) or not _head(fresh): + # The store took the write and did not record it. A 200 here would tell an administrator + # their skill file was saved when it was not — the shape `routes_slack` refuses too. + raise err(503, "store_unavailable", "the harness file was NOT saved") + head = _head(fresh) + return {"agent": agent_id, **_row(fresh), "body": head.get("body") or "", + "atVersion": int(head.get("version") or 0), "history": _version_rows(fresh), + "limits": _limits()} + + +@router.delete("/agents/{agent_id}/harness") +def delete_harness(agent_id: str, path: str = "", session: Session = Depends(require_session)): + """Drop one harness file AND its history. + + ⚠ THIS IS NOT THE THING R8 FORBIDS. R8 forbids a ROLL-BACK implemented as a delete — losing + versions as a side effect of an edit. Deleting a file is a person deciding the file should not + exist, which is a different act with a different button, and a store with no way to remove a + file is one where a typo'd path is permanent. + """ + agent_id = str(agent_id or "") + agent_wall(session, agent_id) + wanted = _clean_path(path) + if wanted not in _files(session.runtime, agent_id): + raise err(404, "no_file", f"this agent has no harness file at {wanted}") + + def _set(cur): + cur = dict(cur or {}) + files = dict(cur.get(agent_id) or {}) if isinstance(cur.get(agent_id), dict) else {} + files.pop(wanted, None) + # An agent with no harness files leaves NO key behind. An empty dict per agent id is how a + # bucket accumulates a row for every agent anybody ever opened the tab on. + if files: + cur[agent_id] = files + else: + cur.pop(agent_id, None) + return cur + + session.runtime.update(HARNESS_KEY, _set, flush="sync") + return {"agent": agent_id, "deleted": wanted, + "files": [_row(rec) for _p, rec in sorted(_files(session.runtime, agent_id).items()) + if isinstance(rec, dict)], + "limits": _limits()} diff --git a/api/routes_alerts.py b/api/routes_alerts.py index 3ff47062772bd99f08e4cd0eb40d25e5e9382404..c651d88cf37431431a9588f38c540b71c89597df 100644 --- a/api/routes_alerts.py +++ b/api/routes_alerts.py @@ -1,668 +1,668 @@ -"""routes_alerts.py — the Alerts module (wave 20, owner item 25, contract C-ALERT). - - GET /api/v1/alerts -> {alerts:[...]} - POST /api/v1/alerts <- {viewId, topic, label?} - DELETE /api/v1/alerts/{alert_id} - POST /api/v1/alerts/{alert_id}/run -> evaluate now (the pane's manual refresh) - GET /api/v1/notifications -> {unread, items:[...]} - POST /api/v1/notifications/read <- {ids:[...]|null, read?:bool} - -The semantics — an alert is a view plus a remembered matched set, a notification is a NEW -ENTRANT, and the first evaluation seeds silently — live in `core.alerts` with the reasoning. -This file owns the two things a route must: WHO may do it, and HOW the view gets evaluated. - -⭐ **THE EVALUATION RUNS AS THE ALERT'S OWNER, NOT AS THE CALLER.** `_run_alert` builds the pool -for `rec['owner']`, never for whoever tripped the write hook. Any other choice leaks: a -full-access admin editing a cell would otherwise evaluate a BU-scoped user's alert over the whole -book, and the notification would name customers that user may not see — a permission leak wearing -a notification's clothes. The owner's own scope is the only correct basis for their alert. - -⚠ **AN ALERT IS NOT A SECOND READ PATH.** It resolves rows through the same -`routes_customers.grid_assembly` / `routes_tables.ut_assembly` the grid uses, so a row that an -alert can see is by construction a row its owner could open. Re-implementing the filter here -would be a second definition of "matches", and those two would drift. -""" -import re - -from fastapi import APIRouter, Body, Depends - -import core.alerts as alerts -from deps import Session, err, require_session - -router = APIRouter(prefix="/api/v1") - -#: The alert-bearing surfaces. `ut_` tables are admitted by prefix, like everywhere else. -_TOPICS = ("customer", "product") - -# ── ⭐⭐ WAVE 32 · T20 · CONTRACT C3 — THE INBOX SHAPE, DERIVED ON READ ──────────────────────── -# -# `GET /notifications` gains `subject`, `kind` and `target` per item (`read` was always there). -# -# ⛔ DERIVED, NEVER STORED, AND THAT IS THE WHOLE OF WHY THIS WAVE EXISTS. Stamping the three -# keys onto the record at write time would give them to notifications minted AFTER the deploy and -# to nothing else — every notification already sitting in every tenant's inbox would open nothing, -# and the feature would be correct in the source and absent from the product -# ([[a-migration-that-runs-on-the-next-write]], D-201). A read-side derivation reaches a -# notification queued last month. It also keeps the store shape out of `core/alerts.py`, which is -# another lane's file this wave — but that is the convenience, not the reason. -# -# ⚠ TWO PRODUCERS WRITE TWO SHAPES into one inbox, and the vocabulary below is what tells them -# apart. `_queue` (a record ENTERED a watched view) sets `topic`+`viewId`. `notify()` sets -# `topic='automation'` and puts the producer's key in `alertId`, leaving `viewId` empty. Deciding -# here means the client branches on ONE field instead of re-deriving the same split. -# -# ⛔ **D-101 IS CLOSED HERE, BY SUBTRACTION.** There was a THIRD shape — `kind='automation_review'` -# + `autoId`, a card arriving at a review stage — and its producer `notify_review` was deleted by -# W27/R3 with the review lanes. `automation_engine.py`'s own tombstone (search `notify_review`) -# records the 2026-08-12 sweep: **no `.py` file anywhere produces one**, while the client branch, -# its route and three gate legs stayed fully alive. D-101's exit condition is *"the client review -# branch is deleted in the same change as any remaining residue, OR `notify_review` gains its real -# caller"* — the residue is zero, so the branch goes. It is not carried into the Inbox: a stored -# review notification (if any survives in a tenant from the wave-23 era) derives as an ordinary -# `alert` with no target, i.e. an honest unclickable row, which is correct — the board it pointed -# at was deleted two waves ago. - -#: C3's `kind` vocabulary. Plain strings on the wire — the client must never union over them -#: (alertsModel's wave-9 law: a client union turns "the server grew a kind" into a dropped row). -NOTIF_KIND_ALERT = "alert" -NOTIF_KIND_AUTOMATION = "automation" -NOTIF_KIND_SHARE = "share" - -#: C3's `target.module` vocabulary, and the automation sub-selection. -TARGET_MODULE_DATABASE = "database" -TARGET_MODULE_AUTOMATION = "automation" -TARGET_TAB_RUNS = "runs" - -#: The topic `notify()` carries for a SHARE (W32-T28 writes it; nothing does yet, and a kind with -#: no producer is a string that reads as a feature — the reason this constant is named here and -#: cited from `routes_shares` rather than typed twice). -SHARE_TOPIC = "share" - -#: `core.alerts.notify`'s default topic for a run outcome. Mirrors `inboxModel.AUTOMATION_TOPIC`. -AUTOMATION_TOPIC = "automation" - -_UT_TOPIC = re.compile(r"ut_[A-Za-z0-9_]+\Z") - - -def route_for_topic(topic): - """A grid SCOPE key -> the registry route that renders it, or None. - - ⛔ THE SAME TABLE AS `alertsModel.routeForTopic`, and the parity is GATED - (`verify_alerts.py`'s vocabulary scan) rather than trusted. The two built-ins are the only - pair that differ — the registry names the surface (`customer_data`) while the grid names the - scope (`customer`) — so a topic passed through as a route sends every click to a page that - does not exist. `None` for anything else: a target this product cannot resolve must be ABSENT - rather than plausible, because an absent target renders as a row that does not pretend to be - clickable, and a wrong one renders as a click that silently goes nowhere. - """ - t = str(topic or "").strip() - if t == "customer": - return "customer_data" - if t == "product": - return "product_data" - if _UT_TOPIC.match(t): - return t - return None - - -def _refusal_code(exc): - """The `error.code` an `HTTPException` raised by `deps.err()` carries, or `""`. - - ⭐ W32-T22. Four refusals travel up the assembly chain — `unknown_table` (404), `forbidden` - (403), `window_required` (409) and `store_not_ready` (503) — and each already names its own - cause. Anything that reduces all four to one word is throwing away the only information the - reader could have acted on. Returns `""` for a plain exception, so a caller can tell - "refused, and here is why" apart from "broke, and we do not know why". - """ - detail = getattr(exc, "detail", None) - if isinstance(detail, dict): - inner = detail.get("error") - if isinstance(inner, dict): - return str(inner.get("code") or "") - return "" - - -def notification_view(item): - """One STORED notification -> the shape the Inbox renders. PURE, and total. - - Never raises and never drops a row: an item it cannot classify comes back as an `alert` with - no `target`, which the client renders as an unclickable row rather than hiding. An inbox that - silently omits what it does not understand is the one failure a reader cannot detect. - """ - if not isinstance(item, dict): - return item - topic = str(item.get("topic") or "").strip() - alert_id = str(item.get("alertId") or "").strip() - - # ⛔ THE ID TEST IS HALF OF EVERY BRANCH, and it is the load-bearing half. A row whose topic - # says `automation` but whose producer key never arrived (a truncated payload, a server - # mid-deploy) would otherwise be handed a target naming NOTHING — a click that appears to work - # and silently does not, which is this repo's most-repeated failure shape. Failing the test - # drops it to the `alert` branch, where `route_for_topic` refuses out loud by answering None. - if topic == AUTOMATION_TOPIC and alert_id: - kind = NOTIF_KIND_AUTOMATION - target = {"module": TARGET_MODULE_AUTOMATION, "id": alert_id, "tab": TARGET_TAB_RUNS} - elif topic == SHARE_TOPIC and alert_id: - # ⭐ W32-T28: the sharer writes `key=` and, for a shared VIEW, - # `row_id=`. - # - # ⛔ `key` IS ALREADY A ROUTE, NOT A RAW OBJECT ID, and the first version of this got it - # wrong in a way worth recording: a shared VIEW put the VIEW's id in `alertId`, so the - # target read `{module: "database", id: "view_42"}` — an instruction to open a database - # called `view_42`. It looked right in the payload and would have opened nothing. The - # producer resolves the object to its topic and hands over the route; this branch only - # shapes what it is given. - kind = NOTIF_KIND_SHARE - row_id = str(item.get("rowId") or "").strip() - target = {"module": TARGET_MODULE_DATABASE, "id": alert_id, - **({"tab": row_id} if row_id else {})} - else: - kind = NOTIF_KIND_ALERT - route = route_for_topic(topic) - view_id = str(item.get("viewId") or "").strip() - target = None if route is None else ( - {"module": TARGET_MODULE_DATABASE, "id": route, - **({"tab": view_id} if view_id else {})}) - - # The email split: `subject` is the HEADER (what this is about — the alert, the automation, - # the database), `label` stays the BODY (what happened — the record that entered, the run - # summary). They were one field, which is why a notification read as a sentence with no - # sender and the pane could not be laid out like mail. - subject = str(item.get("alertLabel") or "").strip() or str(item.get("label") or "").strip() - # ⚠ `kind` is OVERWRITTEN, not merged. There was one stored value (`automation_review`) and it - # is D-101's dead one; leaving it through would give the client two vocabularies for one - # question, which is the defect this wave's item 6 is about in a different file. - # ⭐⭐ W33-T28 (owner: "the Inbox reads like email") — THE SENDER, WHICH DID NOT EXIST. - # - # ⛔ A `verifier` reading the finished wave-32 surface found that the row's sender POSITION was - # occupied by `kindLabel(n.kind)` — the literals "Alert" / "Automation" / "Shared with you" — - # i.e. a CATEGORY standing where a who belongs, and no sender field anywhere on the wire, in - # the model or in the markup. Mail has a from. This is it. - # - # ⚠ IT IS DERIVED HERE, NOT STORED, FOR EVERY KIND BUT ONE — and the exception is the point. - # An alert firing and an automation landing rows have no person behind them; their honest - # sender is the machine that did it, named as the thing the reader recognises. A SHARE has a - # real person, and only the producer knows who: `routes_shares.py` writes it as `actor` and - # this reads it back. ⛔ It is NOT parsed out of the body prose (" shared this with - # you") — a sender recovered by regexing a sentence breaks the first time the sentence is - # reworded, and it would break silently, in the header. - # - # ⚠ FALLS BACK, NEVER BLANK. A share queued BEFORE `actor` existed has none, and a row with an - # empty from column reads as a broken inbox rather than as an old notification. - actor = str(item.get("actor") or "").strip() - if kind == NOTIF_KIND_SHARE: - sender = actor or "A teammate" - elif kind == NOTIF_KIND_AUTOMATION: - # ⛔ "Agents", not "Automation" (W34-T40, corrected at QA 2026-08-17). The client's - # `senderOf` already falls back to `AGENTS_MODULE_LABEL` — but `if (sent) return sent` - # runs FIRST, so this server literal won and every actor-less automation notification - # showed the retired module name in the inbox's From column. - sender = actor or "Agents" - else: - sender = actor or "Alerts" - out = {**item, "read": bool(item.get("read")), "kind": kind, - "subject": subject or "Notification", "sender": sender} - if target is not None: - out["target"] = target - return out - - -def inbox_view(box): - """`core.alerts.inbox()`'s answer, with every item put through {@link notification_view}. - - ⚠ `unread` IS NOT RECOUNTED. It is the ACCOUNT's number and `items` is one page of it; a - recount here would make the badge a function of whatever this page happened to include, which - is the exact defect `alertsModel.parseInbox`'s own header records from the other side. - """ - if not isinstance(box, dict): - return box - items = box.get("items") - if not isinstance(items, list): - return box - # ⭐⭐ W33-T28 / D-208 — THE SERVER'S CLOCK RIDES WITH THE PAGE, and it is what lets the client - # render a mail-shaped stamp ("09:41" today, "Aug 12" beyond) instead of `2026-08-13 09:41`. - # - # ⛔ THE CLIENT MUST NOT READ ITS OWN CLOCK, which is D-208's exit condition word for word and - # is why this key exists rather than a `new Date()` in the browser. `at` is sent as UTC WITH - # its offset (D-18) precisely so every reader sees the same instant; deciding "is this today?" - # against a browser clock would re-introduce the drift the offset exists to remove — a reader a - # day ahead being told an event happened tomorrow [[date-window-vocabulary]]. Both operands - # now come from the same machine. - # ⚠ Same funnel as the enrichment, so the read door and the mark-read door cannot disagree — - # the note two lines up records what happened last time only one of them was enriched. - return {**box, "now": alerts._now_iso(), - "items": [notification_view(n) for n in items]} - - -def _view_by_id(g, view_id): - """One saved view out of an assembly, by id. `None` when there is no such view. - - ⛔⛔ W33-T29 (owner: *"Alert me about new records"* answering "Something went wrong") — THIS - FUNCTION EXISTS BECAUSE TWO CALL SITES BOTH WROTE `(g.get("views") or {}).get(view_id)`, AND - `g["views"] IS A LIST`. `aios_grid.views_from_defs` returns `[{...}]`, `workspace_wire` passes - it straight out and both `ut_assembly` and `grid_assembly` return it unchanged — so `.get` on - it raises `AttributeError`, and `views_from_defs` always returns at least one element, so the - `or {}` never fires. **It raised on EVERY call, on every topic, since wave 20.** - - ⛔ AND THE TWO SITES FAILED DIFFERENTLY, WHICH IS WHY ONLY ONE WAS EVER REPORTED. In - `_require_filtered_view` the raise lands ABOVE the handler's own `try`, so it leaves as a bare - FastAPI 500 and the client's `errorMessage` turns any 5xx into *"Something went wrong on our - side"* — the exact sentence the owner reported (D-107's shape, again: an attribute error above - the guard arrives as plain text rather than as our envelope). In `_evaluate` the identical - line is swallowed by `/notifications`' `except Exception: continue`, so **every stored - view-alert was silently dropped from the Inbox** and nobody had anything to report at all. - One expression, one loud symptom and one silent one. - - ⚠ SO IT IS A FUNCTION, NOT TWO FIXED LINES. Two copies of "find the view" is what let one site - be discussed for three waves while its twin went unnoticed [[one-question-two-normalizers]]. - - ⚠ It accepts a dict too, and that is not defensive noise: `verify_alerts`' door fixture was - keyed `{id: view}` — which is precisely why the gate was green while production raised on - every call. The fixture is moving to the production shape in this same change, and tolerating - both here means a caller that legitimately holds one cannot resurrect the bug. - """ - want = str(view_id or "") - if not want: - return None - views = (g or {}).get("views") - if isinstance(views, dict): - found = views.get(want) - return found if isinstance(found, dict) else None - if not isinstance(views, list): - return None - for v in views: - if isinstance(v, dict) and str(v.get("id") or "") == want: - return v - return None - - -def _topic_or_400(raw): - topic = str(raw or "").strip().lower() - if topic.startswith("ut_") or topic in _TOPICS: - return topic - raise err(400, "bad_topic", f"topic must be one of {', '.join(_TOPICS)} or a ut_ table") - - -def _owner_session(session: Session, owner: str): - """A `Session` for the alert's OWNER (see the module note on why the owner, not the caller). - - ⚠ `Session` exposes `uname`/`admin` as PROPERTIES derived from `user`, not as fields — so an - owner session is built by swapping the `user` RECORD and letting both derive themselves. An - earlier version passed `uname=`/`admin=` to the constructor, which would have raised on the - first write hook of the wave; the properties are the single definition of who a session is, - and going around them is how a session with an admin flag and a non-admin record exists. - - Returns None when the owner is gone or deactivated — their alerts then stop evaluating rather - than evaluating as somebody else, which is the fail-closed direction. - """ - import core.users as users - - if str(owner) == str(session.uname): - return session - rec = (users.registry() or {}).get(str(owner)) - if not isinstance(rec, dict) or not rec.get("active", True): - return None - # `_public` is THE definition of what a session may know about its own account (never a hash - # or a salt) — the same one `routes_auth` uses. Building the dict by hand here would be a - # second definition, and the one that leaks is always the copy. - return Session(tenant=session.tenant, user=users._public(str(owner), rec), - claims=session.claims, runtime=session.runtime) - - -def _evaluate(session: Session, rec: dict, assemblies=None): - """Resolve `rec`'s view over its topic AS THE ALERT'S OWNER, then fold the result in. - - ⭐⭐ W31-T24 — `assemblies` IS A PER-REQUEST MEMO, KEYED `(topic, owner)`, and it is the whole - of this ticket's server half. `/notifications` re-evaluates EVERY alert inline on read and each - one built a FULL assembly — the pool, the workspace, `rows_from_pool` over every row. Two - alerts on one view built that table twice; ten built it ten times. Nothing dedupes them, - because each `_evaluate` was a closed call. - ⚠ `(topic, owner)` and not `topic`: the assembly is built as the alert's OWNER (see the module - note — evaluating a BU-scoped user's alert on a full-access admin's pool is a permission leak - wearing a notification's clothes), so two owners on one topic are two DIFFERENT tables and - must never share an entry. Getting that key wrong is the one way this optimisation could leak. - ⚠ Passing nothing keeps the old behaviour exactly, which is what the create/run doors want: - they evaluate ONE alert and a memo for a single call is pure overhead. - """ - import aios_grid - from harness import filter_eval - - owner_sess = _owner_session(session, rec.get("owner")) - if owner_sess is None: - return {"skipped": "owner_unavailable"} - topic = str(rec.get("topic") or "") - memo_key = (topic, str(owner_sess.uname)) - g = assemblies.get(memo_key) if isinstance(assemblies, dict) else None - if g is None: - try: - if topic.startswith("ut_"): - from routes_tables import ut_assembly - # ⛔ `consume_corrections=False`, and the default was a REAL BUG, not a tidy-up. - # `ut_assembly` defaults it True, so every `/notifications` read CONSUMED the - # one-shot field-name correction acks for every `ut_` topic that has an alert — - # taking them from the `/workspace` refresh that exists to show them to the person - # who made the edit. The customer branch below has always passed False; this one - # inherited a default nobody re-read. An inbox poll must never consume a one-shot. - g = ut_assembly(owner_sess, topic, - storage_key=f"{owner_sess.tenant}:{topic}:{owner_sess.uname}", - consume_corrections=False) - else: - from routes_customers import grid_assembly - g = grid_assembly(owner_sess, scope=topic, consume_corrections=False) - except Exception as e: # noqa: BLE001 - # ⭐ W32-T22 — SKIPPING IS FINE HERE; SKIPPING ANONYMOUSLY IS NOT. This one must not - # raise (one bad alert cannot empty an inbox), so unlike `_require_filtered_view` it - # keeps a blanket catch — but it now reports the refusal's OWN code where there is - # one. `type(e).__name__` said `HTTPException` for four different causes, and - # `lastError` is the only place a user ever learns why an alert stopped firing. - # - # ⚠ `with_rows=True` STAYS on this path, deliberately: unlike the create door, an - # evaluation genuinely needs the rows to run the filter over. So an alert on a - # read-through grid is created (T22) and then skips at evaluation with - # `window_required` naming why — which is D-184's remaining half, and it is a - # SENTENCE now rather than silence. - return {"skipped": _refusal_code(e) or "unavailable", "detail": type(e).__name__} - if isinstance(assemblies, dict): - assemblies[memo_key] = g - - view = _view_by_id(g, rec.get("viewId")) - if not isinstance(view, dict): - # Deleted, or un-shared out from under the alert. Say so on the RECORD rather than - # deleting the alert: an alert that silently vanishes is indistinguishable from one that - # never fires, and the user cannot debug what is not there. - return {"skipped": "view_missing"} - - # The SAME row build the grid and `/customers` use — `rows_from_pool` is what puts derived - # and overlay values on a row. Evaluating a filter against raw pool dicts would silently - # never match any condition on a user-created or measure column. - # - # ⛔⛔ AND "THE SAME ROW BUILD" WAS NOT TRUE, WHICH MADE EVERY ALERT ON A `ut_*` DATABASE BLIND - # TO IMPORTED DATA. Found by a verifier driving one real assembly through both paths. - # - # `routes_tables.table_rows` — the grid the person is looking at — merges the DEFINITION rows - # underneath the overlay ("base first, overlay wins"; that merge is itself the fix for owner - # item 3, *"it all got reseted"*). `_evaluate` is a second copy of that read and never got it: - # it handed `ws['overlays']` to `rows_from_pool` raw, so for a `ut_*` table every base cell - # evaluated as BLANK. Measured on one assembly, same view, same rows: - # rows_src state='unpaid' / 'paid' - # _evaluate saw state='' / '' ⇐ every base cell blank - # the GRID saw state='unpaid' / 'paid' - # so `state eq unpaid` matched NOTHING while the view showed one row, and `state isEmpty` - # matched EVERYTHING while the view showed none. **The alert did not merely miss rows — it - # inverted.** End to end: a row whose value arrived by import, automation, paste or the create - # door never fired; only a value typed as a hand EDIT did. - # - # ⚠ Scope, so nobody widens the fix past its cause: materialised `ut_*` tables are hit; - # `customer`/`product` are not (their fields are `source: "odoo"` and read off `rows_src`); - # `ut_odoo_*` never reaches here (`with_rows=True` refuses first and returns - # `skipped: window_required`). - # ⛔ ORDER IS LOAD-BEARING AND IS THE GRID'S: base underneath, overlay ON TOP. Inverting it - # would let a stale definition value shadow an edit the user has just made — the same defect - # `table_rows`' own note records, arriving from the other side. - _ov = (g.get("ws") or {}).get("overlays") or {} - _merged = {} - for _r in g["rows_src"]: - _pid = str(_r.get("pid")) - _cells = {k: v for k, v in _r.items() if k != "pid"} - _o = _ov.get(_pid) - if isinstance(_o, dict): - _cells.update(_o) - _merged[_pid] = _cells - rows = aios_grid.rows_from_pool(g["rows_src"], g["fields"], _merged, - derived=g.get("derived")) - config = view.get("config") or view - ctx = filter_eval.EvalCtx( - cohort_sets={str(k): {str(p) for p in (v.get("memberPids") or ())} - for k, v in (g.get("lists") or {}).items() if isinstance(v, dict)}, - measure_sets=g.get("measure_sets") or {}, - today=g.get("today")) - pids = filter_eval.visible_pids(config.get("filters") or [], rows, g["fields"], ctx, - member_pids=config.get("memberPids")) - labels = {str(r.get("pid")): str(r.get("name") or r.get("pid")) for r in rows} - return alerts.evaluate(rec.get("id"), [str(p) for p in pids], - labels=labels, partial=False, st=session.runtime) - - -@router.get("/alerts") -def list_alerts(session: Session = Depends(require_session)): - return {"alerts": alerts.list_alerts(user=session.uname, is_admin=session.admin, - st=session.runtime)} - - -@router.post("/alerts") -def create_alert(body: dict = Body(default=None), session: Session = Depends(require_session)): - body = body or {} - view_id = str(body.get("viewId") or "").strip() - if not view_id: - raise err(400, "bad_view", "an alert needs the id of the view it watches") - topic = _topic_or_400(body.get("topic")) - _require_filtered_view(session, topic, view_id) - import uuid - aid = f"al_{uuid.uuid4().hex[:12]}" - rec = alerts.create(aid, view_id=view_id, topic=topic, owner=session.uname, - label=body.get("label") or "", st=session.runtime) - # SEED IMMEDIATELY, so the alert starts from "everything currently matching is old news". - # Deferring this to the first write hook would mean the next edit announces the whole view. - outcome = _evaluate(session, rec) - return {"alert": {**rec, "seeded": True}, "first": outcome} - - -def _require_filtered_view(session: Session, topic: str, view_id: str): - """400 unless `view_id` exists on `topic` AND actually narrows something. - - ⛔ AN ALERT ON AN UNFILTERED VIEW IS SILENTLY INCAPABLE OF ALERTING, which is worse than one - that is refused. `filter_eval` treats an inactive tree as "no narrowing, every row shows" - (`visible_pids`'s own rule), so such an alert seeds with the entire table and can never see an - entrant again — there is nothing left to enter. The owner's words are *"when a Record gets - into that Filter's criteria"*: no criteria, no alert, and said at creation rather than - discovered by never being notified. - - `is_rule_active` is the SAME activeness predicate the engine and the column tints use — a - half-typed rule is not a filter, and this must agree with what actually narrows or it would - accept a view whose one rule the engine then ignores. - - ⭐⭐ WAVE 32 · T22 (owner item 17) — THIS FUNCTION WAS THE ERROR. Two defects, stacked, and - the second one hid the first. - - (1) **IT ASKED FOR EVERY ROW OF A TABLE IT NEVER LOOKS AT.** The only thing read below is - `g["views"]`. `ut_assembly` defaults `with_rows=True`, so creating an alert on a - read-through grid built the whole pool — and `scoped_pool` refuses that with - `409 window_required` over 963,783 rows, exactly as it is supposed to. `with_rows=False` - (W31-T20's flag, built for precisely this) answers the same question with `scoped_pids`, - runs the SAME `_defn_or_refuse` wall, and does not refuse. **That is D-184's create half, - closed** — an alert on a read-through grid can now be made at all. - (2) **A BLANKET `except Exception` TURNED EVERY NAMED REFUSAL INTO A 503.** `HTTPException` - is an `Exception`, so `404 unknown_table`, `403 forbidden`, `409 window_required` and - `503 store_not_ready` — four refusals that each say what is wrong — were all replaced by - *"the table is unavailable — try again in a moment"*. ⛔ AND THAT SENTENCE NEVER REACHED - A USER EITHER: `alertsApi.errorMessage` discards the text of any status ≥ 500 by design - (a 5xx body is the server's internals), substituting *"Something went wrong on our - side."* — which is the owner's screenshot, word for word. A knowable cause returned as a - 5xx is invisible by construction, so re-wording the 503 could never have fixed this. - ⚠ The except is narrowed, not deleted: an UNEXPECTED failure is still a 503, because that is - honest. What it may no longer do is catch a refusal that already knows its own name. - """ - from fastapi import HTTPException - - from harness import filter_eval - - try: - if topic.startswith("ut_"): - from routes_tables import ut_assembly - # ⚠ `consume_corrections=False` — the customer branch has always passed it and this - # one inherited a default nobody re-read. Creating an alert must not eat the one-shot - # field-name correction acks belonging to the `/workspace` refresh that exists to show - # them to the person who made the edit. Same defect `_evaluate`'s header records. - g = ut_assembly(session, topic, - storage_key=f"{session.tenant}:{topic}:{session.uname}", - consume_corrections=False, with_rows=False) - else: - from routes_customers import grid_assembly - g = grid_assembly(session, scope=topic, consume_corrections=False) - except HTTPException: - raise # it already names its own cause - except Exception as e: # noqa: BLE001 - # Genuinely unexpected. Still a 503, and now it carries the exception TYPE — without it, - # the one path that reaches this branch is also the one path with nothing to debug from. - raise err(503, "unavailable", - f"the table could not be read ({type(e).__name__}) — try again in a moment") - view = _view_by_id(g, view_id) - if not isinstance(view, dict): - raise err(404, "no_view", "that view does not exist on this table") - nodes, _conj = filter_eval.tree_parts((view.get("config") or view).get("filters") or []) - - # ⛔⛔ WAVE 33 · T29 — **CORRECTION: THE BLOCK BELOW IS TRUE ABOUT THE CODE AND FALSE ABOUT - # PRODUCTION, AND IT MUST BE READ SECOND.** It claims the missing-argument `TypeError` "IS - # owner item 17" — the owner's *"Something went wrong"*. It was not, and it could not have - # been: at `cbcf005`, the build the owner was using, the dict-read on `views` sat ~10 lines - # ABOVE this call and raised `AttributeError` on EVERY request, so the walk never reached the - # leaf and the arity bug was unreachable. `_view_by_id`'s own header records that fix. - # - # ⚠ WHY THE STALE PARAGRAPH STAYS RATHER THAN GETTING DELETED: the arity bug was real, the - # fix was right, and the three reasons it hid are the most transferable thing in this file. - # What was wrong is only its CLAIM TO BE THE CAUSE. Two comment blocks in one function each - # naming themselves as the origin of the same screenshot are mutually exclusive, and the next - # reader believes whichever they meet first — which is why this correction sits above rather - # than below. Caught by a verifier that read the SHIPPED file at the deployed commit instead - # of the working tree. [[grep-output-is-not-source]] - # - # ⛔⛔ WAVE 32 · T22 — **THE CALL BELOW WAS MISSING AN ARGUMENT** (and wave 32 believed, wrongly, - # that this was owner item 17 — see the correction directly above). - # - # `is_rule_active(rule, columns)` takes TWO parameters (`harness/filter_sql.py`; every other - # caller in the repo passes both). This one passed ONE, so the moment the walk reached a LEAF - # rule it raised `TypeError: is_rule_active() missing 1 required positional argument`. - # - # ⚠ READ WHAT THAT MEANS BEFORE FIXING ANYTHING ELSE: the walk only reaches a leaf when the - # view HAS a condition — and a view with a condition is the only kind an alert is allowed on. - # A view with no filters yields an empty `nodes`, so `_any_active` returns False without ever - # calling this, and the reader gets the honest 400 `no_filter`. **So the only path that - # worked was the refusal path: "Alert me about new records" had never once created an alert - # on a filtered view.** ⛔ And the raise lands OUTSIDE the `try` above, so it was not even the - # 503 — it was a bare FastAPI 500, which `alertsApi.errorMessage` renders as *"Something went - # wrong on our side. Try again in a moment."*, the owner's screenshot word for word. - # - # ⚠ THREE THINGS HID IT, and they are worth more than the fix. (1) Python does not check - # arity until the line RUNS, and this line runs only on the success path of a feature whose - # every test exercised its refusals. (2) The `no_filter` 400 above it is a real, correct, - # well-tested refusal, so the door looked alive. (3) `verify_alerts.py` asserts the refusal - # (`no_filter` reaches the user) and the transport — never a creation. A gate can be green, - # thorough and honest about everything except the one path the feature exists for. - # - # `_columns_map` is the DEFINITION of fields -> the membership set `is_rule_active` looks a - # column up in; building a second dict here would be a second answer to one question, which - # is this wave's other headline defect in a different file. Its leading underscore is a real - # smell and is BOOKED (PENDING, mailbox/C.md) rather than worked around. - columns = filter_eval._columns_map(g.get("fields") or []) - - def _any_active(ns): - for n in ns or (): - if isinstance(n, dict) and isinstance(n.get("children"), list): - if _any_active(n["children"]): - return True - elif filter_eval.is_rule_active(n, columns): - return True - return False - - if not _any_active(nodes): - raise err(400, "no_filter", - "this view has no active filter, so no record can ever ENTER it — add a " - "condition to the view first, then create the alert") - - -@router.delete("/alerts/{alert_id}") -def delete_alert(alert_id: str, session: Session = Depends(require_session)): - rec = next((r for r in alerts.list_alerts(st=session.runtime) - if str(r.get("id")) == str(alert_id)), None) - if rec is None: - raise err(404, "no_alert", "that alert does not exist") - if str(rec.get("owner")) != str(session.uname) and not session.admin: - raise err(403, "not_yours", "only the alert's owner (or an administrator) can delete it") - alerts.delete(alert_id, st=session.runtime) - return {"ok": True} - - -@router.post("/alerts/{alert_id}/run") -def run_alert(alert_id: str, session: Session = Depends(require_session)): - rec = next((r for r in alerts.list_alerts(user=session.uname, is_admin=session.admin, - st=session.runtime) - if str(r.get("id")) == str(alert_id)), None) - if rec is None: - raise err(404, "no_alert", "that alert does not exist") - return _evaluate(session, rec) - - -@router.get("/notifications") -def notifications(session: Session = Depends(require_session)): - """The inbox — RE-EVALUATED on read, which is a deliberate design choice. - - ⭐ A-S1-2 RESOLVED THE OTHER WAY, and the reason is structural rather than a shortcut. The - plan was a push hook: the automation engine calls `after_write` when it lands rows. But - `run_async` runs on a BACKGROUND THREAD with no `Session` in scope, and an alert must be - evaluated as its OWNER (see `_evaluate`) — so a push hook would have to mint a session inside - a worker thread from a tenant runtime, which is exactly the kind of ad-hoc identity - construction that leaks scope. - - Pulling on read has none of that: the caller IS a session, the assemblies are already - scope-cached, and the user cannot observe the difference — an inbox is only ever read by - someone opening it. The cost is that a notification is minted when you LOOK rather than when - the row landed, so the `at` stamp is detection time, not arrival time. - - `after_write` stays exported for the day the engine can hand over a real identity. - - ⭐⭐ W31-T24 — ONE ASSEMBLY PER (TOPIC, OWNER), NOT ONE PER ALERT. - ⛔ MEASURED FIRST, AND THE MEASUREMENT CORRECTS AN EARLIER READING OF IT. This route is - **20 ms in-process and 3,280 ms live** on tenant #0 — but tenant #0 has **ZERO alerts** - (censused 2026-08-12), so the 20 ms is an EMPTY LOOP and says nothing at all about what the - re-evaluation costs. The live 3,280 ms is the two store reads either side of that loop. So the - body below is not slow today; it is UNEXERCISED, and every alert a tenant creates adds a whole - grid assembly to an inbox poll. The memo turns O(alerts) into O(distinct topic × owner), which - is the difference between "fine" and "three seconds per alert" the day somebody uses the - feature. ⚠ Making the read cheap by evaluating LESS is the obvious wrong fix and is not what - this does: every alert is still evaluated, against the same rows, in the same order. - """ - assemblies = {} - for rec in alerts.list_alerts(user=session.uname, is_admin=False, st=session.runtime): - try: - _evaluate(session, rec, assemblies=assemblies) - except Exception: # noqa: BLE001 - continue # one bad alert must not empty the pane - # ⭐ W32-T20 (C3): every item leaves through `inbox_view`, so a notification queued before - # this wave carries a `target` too. See `notification_view`'s header for why it is derived. - return inbox_view(alerts.inbox(session.uname, st=session.runtime)) - - -@router.post("/notifications/read") -def read_notifications(body: dict = Body(default=None), - session: Session = Depends(require_session)): - body = body or {} - ids = body.get("ids") - if ids is not None and not isinstance(ids, list): - raise err(400, "bad_ids", "ids must be a list, or null to mark every notification") - # ⚠ THE SAME ENRICHMENT ON BOTH DOORS. `mark_read` returns a fresh inbox, and the Inbox - # module re-renders from it — an un-enriched answer here would strip `target` off every row - # the moment somebody marked one read, i.e. the feature would work until first use. - return inbox_view(alerts.mark_read(session.uname, ids, read=bool(body.get("read", True)), - st=session.runtime)) - - -def after_write(session: Session, topic_key: str): - """THE WRITE HOOK — call after a write that could change what a view matches. - - Exported as a plain function (not a route) so `core.grid_events`' callers and S2's automation - upserts reach it the same way. It never raises: an alert evaluation failing must not fail the - edit that triggered it. - - ⭐ W31-T24 — it shares `/notifications`' memo shape for the same reason: a write that changes - one view can trip several alerts on the SAME topic, and each would otherwise rebuild the table. - ⚠ STILL ZERO PRODUCTION CALLERS (W31-T24 confirmed it; the route docstring above says why the - push hook was resolved the other way). Booked rather than wired: minting a session inside the - engine's worker thread is the ad-hoc identity construction this file exists to avoid. - """ - try: - assemblies = {} - return alerts.after_write(topic_key, st=session.runtime, - runner=lambda rec: _evaluate(session, rec, - assemblies=assemblies)) - except Exception: # noqa: BLE001 - return {"evaluated": 0} +"""routes_alerts.py — the Alerts module (wave 20, owner item 25, contract C-ALERT). + + GET /api/v1/alerts -> {alerts:[...]} + POST /api/v1/alerts <- {viewId, topic, label?} + DELETE /api/v1/alerts/{alert_id} + POST /api/v1/alerts/{alert_id}/run -> evaluate now (the pane's manual refresh) + GET /api/v1/notifications -> {unread, items:[...]} + POST /api/v1/notifications/read <- {ids:[...]|null, read?:bool} + +The semantics — an alert is a view plus a remembered matched set, a notification is a NEW +ENTRANT, and the first evaluation seeds silently — live in `core.alerts` with the reasoning. +This file owns the two things a route must: WHO may do it, and HOW the view gets evaluated. + +⭐ **THE EVALUATION RUNS AS THE ALERT'S OWNER, NOT AS THE CALLER.** `_run_alert` builds the pool +for `rec['owner']`, never for whoever tripped the write hook. Any other choice leaks: a +full-access admin editing a cell would otherwise evaluate a BU-scoped user's alert over the whole +book, and the notification would name customers that user may not see — a permission leak wearing +a notification's clothes. The owner's own scope is the only correct basis for their alert. + +⚠ **AN ALERT IS NOT A SECOND READ PATH.** It resolves rows through the same +`routes_customers.grid_assembly` / `routes_tables.ut_assembly` the grid uses, so a row that an +alert can see is by construction a row its owner could open. Re-implementing the filter here +would be a second definition of "matches", and those two would drift. +""" +import re + +from fastapi import APIRouter, Body, Depends + +import core.alerts as alerts +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + +#: The alert-bearing surfaces. `ut_` tables are admitted by prefix, like everywhere else. +_TOPICS = ("customer", "product") + +# ── ⭐⭐ WAVE 32 · T20 · CONTRACT C3 — THE INBOX SHAPE, DERIVED ON READ ──────────────────────── +# +# `GET /notifications` gains `subject`, `kind` and `target` per item (`read` was always there). +# +# ⛔ DERIVED, NEVER STORED, AND THAT IS THE WHOLE OF WHY THIS WAVE EXISTS. Stamping the three +# keys onto the record at write time would give them to notifications minted AFTER the deploy and +# to nothing else — every notification already sitting in every tenant's inbox would open nothing, +# and the feature would be correct in the source and absent from the product +# ([[a-migration-that-runs-on-the-next-write]], D-201). A read-side derivation reaches a +# notification queued last month. It also keeps the store shape out of `core/alerts.py`, which is +# another lane's file this wave — but that is the convenience, not the reason. +# +# ⚠ TWO PRODUCERS WRITE TWO SHAPES into one inbox, and the vocabulary below is what tells them +# apart. `_queue` (a record ENTERED a watched view) sets `topic`+`viewId`. `notify()` sets +# `topic='automation'` and puts the producer's key in `alertId`, leaving `viewId` empty. Deciding +# here means the client branches on ONE field instead of re-deriving the same split. +# +# ⛔ **D-101 IS CLOSED HERE, BY SUBTRACTION.** There was a THIRD shape — `kind='automation_review'` +# + `autoId`, a card arriving at a review stage — and its producer `notify_review` was deleted by +# W27/R3 with the review lanes. `automation_engine.py`'s own tombstone (search `notify_review`) +# records the 2026-08-12 sweep: **no `.py` file anywhere produces one**, while the client branch, +# its route and three gate legs stayed fully alive. D-101's exit condition is *"the client review +# branch is deleted in the same change as any remaining residue, OR `notify_review` gains its real +# caller"* — the residue is zero, so the branch goes. It is not carried into the Inbox: a stored +# review notification (if any survives in a tenant from the wave-23 era) derives as an ordinary +# `alert` with no target, i.e. an honest unclickable row, which is correct — the board it pointed +# at was deleted two waves ago. + +#: C3's `kind` vocabulary. Plain strings on the wire — the client must never union over them +#: (alertsModel's wave-9 law: a client union turns "the server grew a kind" into a dropped row). +NOTIF_KIND_ALERT = "alert" +NOTIF_KIND_AUTOMATION = "automation" +NOTIF_KIND_SHARE = "share" + +#: C3's `target.module` vocabulary, and the automation sub-selection. +TARGET_MODULE_DATABASE = "database" +TARGET_MODULE_AUTOMATION = "automation" +TARGET_TAB_RUNS = "runs" + +#: The topic `notify()` carries for a SHARE (W32-T28 writes it; nothing does yet, and a kind with +#: no producer is a string that reads as a feature — the reason this constant is named here and +#: cited from `routes_shares` rather than typed twice). +SHARE_TOPIC = "share" + +#: `core.alerts.notify`'s default topic for a run outcome. Mirrors `inboxModel.AUTOMATION_TOPIC`. +AUTOMATION_TOPIC = "automation" + +_UT_TOPIC = re.compile(r"ut_[A-Za-z0-9_]+\Z") + + +def route_for_topic(topic): + """A grid SCOPE key -> the registry route that renders it, or None. + + ⛔ THE SAME TABLE AS `alertsModel.routeForTopic`, and the parity is GATED + (`verify_alerts.py`'s vocabulary scan) rather than trusted. The two built-ins are the only + pair that differ — the registry names the surface (`customer_data`) while the grid names the + scope (`customer`) — so a topic passed through as a route sends every click to a page that + does not exist. `None` for anything else: a target this product cannot resolve must be ABSENT + rather than plausible, because an absent target renders as a row that does not pretend to be + clickable, and a wrong one renders as a click that silently goes nowhere. + """ + t = str(topic or "").strip() + if t == "customer": + return "customer_data" + if t == "product": + return "product_data" + if _UT_TOPIC.match(t): + return t + return None + + +def _refusal_code(exc): + """The `error.code` an `HTTPException` raised by `deps.err()` carries, or `""`. + + ⭐ W32-T22. Four refusals travel up the assembly chain — `unknown_table` (404), `forbidden` + (403), `window_required` (409) and `store_not_ready` (503) — and each already names its own + cause. Anything that reduces all four to one word is throwing away the only information the + reader could have acted on. Returns `""` for a plain exception, so a caller can tell + "refused, and here is why" apart from "broke, and we do not know why". + """ + detail = getattr(exc, "detail", None) + if isinstance(detail, dict): + inner = detail.get("error") + if isinstance(inner, dict): + return str(inner.get("code") or "") + return "" + + +def notification_view(item): + """One STORED notification -> the shape the Inbox renders. PURE, and total. + + Never raises and never drops a row: an item it cannot classify comes back as an `alert` with + no `target`, which the client renders as an unclickable row rather than hiding. An inbox that + silently omits what it does not understand is the one failure a reader cannot detect. + """ + if not isinstance(item, dict): + return item + topic = str(item.get("topic") or "").strip() + alert_id = str(item.get("alertId") or "").strip() + + # ⛔ THE ID TEST IS HALF OF EVERY BRANCH, and it is the load-bearing half. A row whose topic + # says `automation` but whose producer key never arrived (a truncated payload, a server + # mid-deploy) would otherwise be handed a target naming NOTHING — a click that appears to work + # and silently does not, which is this repo's most-repeated failure shape. Failing the test + # drops it to the `alert` branch, where `route_for_topic` refuses out loud by answering None. + if topic == AUTOMATION_TOPIC and alert_id: + kind = NOTIF_KIND_AUTOMATION + target = {"module": TARGET_MODULE_AUTOMATION, "id": alert_id, "tab": TARGET_TAB_RUNS} + elif topic == SHARE_TOPIC and alert_id: + # ⭐ W32-T28: the sharer writes `key=` and, for a shared VIEW, + # `row_id=`. + # + # ⛔ `key` IS ALREADY A ROUTE, NOT A RAW OBJECT ID, and the first version of this got it + # wrong in a way worth recording: a shared VIEW put the VIEW's id in `alertId`, so the + # target read `{module: "database", id: "view_42"}` — an instruction to open a database + # called `view_42`. It looked right in the payload and would have opened nothing. The + # producer resolves the object to its topic and hands over the route; this branch only + # shapes what it is given. + kind = NOTIF_KIND_SHARE + row_id = str(item.get("rowId") or "").strip() + target = {"module": TARGET_MODULE_DATABASE, "id": alert_id, + **({"tab": row_id} if row_id else {})} + else: + kind = NOTIF_KIND_ALERT + route = route_for_topic(topic) + view_id = str(item.get("viewId") or "").strip() + target = None if route is None else ( + {"module": TARGET_MODULE_DATABASE, "id": route, + **({"tab": view_id} if view_id else {})}) + + # The email split: `subject` is the HEADER (what this is about — the alert, the automation, + # the database), `label` stays the BODY (what happened — the record that entered, the run + # summary). They were one field, which is why a notification read as a sentence with no + # sender and the pane could not be laid out like mail. + subject = str(item.get("alertLabel") or "").strip() or str(item.get("label") or "").strip() + # ⚠ `kind` is OVERWRITTEN, not merged. There was one stored value (`automation_review`) and it + # is D-101's dead one; leaving it through would give the client two vocabularies for one + # question, which is the defect this wave's item 6 is about in a different file. + # ⭐⭐ W33-T28 (owner: "the Inbox reads like email") — THE SENDER, WHICH DID NOT EXIST. + # + # ⛔ A `verifier` reading the finished wave-32 surface found that the row's sender POSITION was + # occupied by `kindLabel(n.kind)` — the literals "Alert" / "Automation" / "Shared with you" — + # i.e. a CATEGORY standing where a who belongs, and no sender field anywhere on the wire, in + # the model or in the markup. Mail has a from. This is it. + # + # ⚠ IT IS DERIVED HERE, NOT STORED, FOR EVERY KIND BUT ONE — and the exception is the point. + # An alert firing and an automation landing rows have no person behind them; their honest + # sender is the machine that did it, named as the thing the reader recognises. A SHARE has a + # real person, and only the producer knows who: `routes_shares.py` writes it as `actor` and + # this reads it back. ⛔ It is NOT parsed out of the body prose (" shared this with + # you") — a sender recovered by regexing a sentence breaks the first time the sentence is + # reworded, and it would break silently, in the header. + # + # ⚠ FALLS BACK, NEVER BLANK. A share queued BEFORE `actor` existed has none, and a row with an + # empty from column reads as a broken inbox rather than as an old notification. + actor = str(item.get("actor") or "").strip() + if kind == NOTIF_KIND_SHARE: + sender = actor or "A teammate" + elif kind == NOTIF_KIND_AUTOMATION: + # ⛔ "Agents", not "Automation" (W34-T40, corrected at QA 2026-08-17). The client's + # `senderOf` already falls back to `AGENTS_MODULE_LABEL` — but `if (sent) return sent` + # runs FIRST, so this server literal won and every actor-less automation notification + # showed the retired module name in the inbox's From column. + sender = actor or "Agents" + else: + sender = actor or "Alerts" + out = {**item, "read": bool(item.get("read")), "kind": kind, + "subject": subject or "Notification", "sender": sender} + if target is not None: + out["target"] = target + return out + + +def inbox_view(box): + """`core.alerts.inbox()`'s answer, with every item put through {@link notification_view}. + + ⚠ `unread` IS NOT RECOUNTED. It is the ACCOUNT's number and `items` is one page of it; a + recount here would make the badge a function of whatever this page happened to include, which + is the exact defect `alertsModel.parseInbox`'s own header records from the other side. + """ + if not isinstance(box, dict): + return box + items = box.get("items") + if not isinstance(items, list): + return box + # ⭐⭐ W33-T28 / D-208 — THE SERVER'S CLOCK RIDES WITH THE PAGE, and it is what lets the client + # render a mail-shaped stamp ("09:41" today, "Aug 12" beyond) instead of `2026-08-13 09:41`. + # + # ⛔ THE CLIENT MUST NOT READ ITS OWN CLOCK, which is D-208's exit condition word for word and + # is why this key exists rather than a `new Date()` in the browser. `at` is sent as UTC WITH + # its offset (D-18) precisely so every reader sees the same instant; deciding "is this today?" + # against a browser clock would re-introduce the drift the offset exists to remove — a reader a + # day ahead being told an event happened tomorrow [[date-window-vocabulary]]. Both operands + # now come from the same machine. + # ⚠ Same funnel as the enrichment, so the read door and the mark-read door cannot disagree — + # the note two lines up records what happened last time only one of them was enriched. + return {**box, "now": alerts._now_iso(), + "items": [notification_view(n) for n in items]} + + +def _view_by_id(g, view_id): + """One saved view out of an assembly, by id. `None` when there is no such view. + + ⛔⛔ W33-T29 (owner: *"Alert me about new records"* answering "Something went wrong") — THIS + FUNCTION EXISTS BECAUSE TWO CALL SITES BOTH WROTE `(g.get("views") or {}).get(view_id)`, AND + `g["views"] IS A LIST`. `aios_grid.views_from_defs` returns `[{...}]`, `workspace_wire` passes + it straight out and both `ut_assembly` and `grid_assembly` return it unchanged — so `.get` on + it raises `AttributeError`, and `views_from_defs` always returns at least one element, so the + `or {}` never fires. **It raised on EVERY call, on every topic, since wave 20.** + + ⛔ AND THE TWO SITES FAILED DIFFERENTLY, WHICH IS WHY ONLY ONE WAS EVER REPORTED. In + `_require_filtered_view` the raise lands ABOVE the handler's own `try`, so it leaves as a bare + FastAPI 500 and the client's `errorMessage` turns any 5xx into *"Something went wrong on our + side"* — the exact sentence the owner reported (D-107's shape, again: an attribute error above + the guard arrives as plain text rather than as our envelope). In `_evaluate` the identical + line is swallowed by `/notifications`' `except Exception: continue`, so **every stored + view-alert was silently dropped from the Inbox** and nobody had anything to report at all. + One expression, one loud symptom and one silent one. + + ⚠ SO IT IS A FUNCTION, NOT TWO FIXED LINES. Two copies of "find the view" is what let one site + be discussed for three waves while its twin went unnoticed [[one-question-two-normalizers]]. + + ⚠ It accepts a dict too, and that is not defensive noise: `verify_alerts`' door fixture was + keyed `{id: view}` — which is precisely why the gate was green while production raised on + every call. The fixture is moving to the production shape in this same change, and tolerating + both here means a caller that legitimately holds one cannot resurrect the bug. + """ + want = str(view_id or "") + if not want: + return None + views = (g or {}).get("views") + if isinstance(views, dict): + found = views.get(want) + return found if isinstance(found, dict) else None + if not isinstance(views, list): + return None + for v in views: + if isinstance(v, dict) and str(v.get("id") or "") == want: + return v + return None + + +def _topic_or_400(raw): + topic = str(raw or "").strip().lower() + if topic.startswith("ut_") or topic in _TOPICS: + return topic + raise err(400, "bad_topic", f"topic must be one of {', '.join(_TOPICS)} or a ut_ table") + + +def _owner_session(session: Session, owner: str): + """A `Session` for the alert's OWNER (see the module note on why the owner, not the caller). + + ⚠ `Session` exposes `uname`/`admin` as PROPERTIES derived from `user`, not as fields — so an + owner session is built by swapping the `user` RECORD and letting both derive themselves. An + earlier version passed `uname=`/`admin=` to the constructor, which would have raised on the + first write hook of the wave; the properties are the single definition of who a session is, + and going around them is how a session with an admin flag and a non-admin record exists. + + Returns None when the owner is gone or deactivated — their alerts then stop evaluating rather + than evaluating as somebody else, which is the fail-closed direction. + """ + import core.users as users + + if str(owner) == str(session.uname): + return session + rec = (users.registry() or {}).get(str(owner)) + if not isinstance(rec, dict) or not rec.get("active", True): + return None + # `_public` is THE definition of what a session may know about its own account (never a hash + # or a salt) — the same one `routes_auth` uses. Building the dict by hand here would be a + # second definition, and the one that leaks is always the copy. + return Session(tenant=session.tenant, user=users._public(str(owner), rec), + claims=session.claims, runtime=session.runtime) + + +def _evaluate(session: Session, rec: dict, assemblies=None): + """Resolve `rec`'s view over its topic AS THE ALERT'S OWNER, then fold the result in. + + ⭐⭐ W31-T24 — `assemblies` IS A PER-REQUEST MEMO, KEYED `(topic, owner)`, and it is the whole + of this ticket's server half. `/notifications` re-evaluates EVERY alert inline on read and each + one built a FULL assembly — the pool, the workspace, `rows_from_pool` over every row. Two + alerts on one view built that table twice; ten built it ten times. Nothing dedupes them, + because each `_evaluate` was a closed call. + ⚠ `(topic, owner)` and not `topic`: the assembly is built as the alert's OWNER (see the module + note — evaluating a BU-scoped user's alert on a full-access admin's pool is a permission leak + wearing a notification's clothes), so two owners on one topic are two DIFFERENT tables and + must never share an entry. Getting that key wrong is the one way this optimisation could leak. + ⚠ Passing nothing keeps the old behaviour exactly, which is what the create/run doors want: + they evaluate ONE alert and a memo for a single call is pure overhead. + """ + import aios_grid + from harness import filter_eval + + owner_sess = _owner_session(session, rec.get("owner")) + if owner_sess is None: + return {"skipped": "owner_unavailable"} + topic = str(rec.get("topic") or "") + memo_key = (topic, str(owner_sess.uname)) + g = assemblies.get(memo_key) if isinstance(assemblies, dict) else None + if g is None: + try: + if topic.startswith("ut_"): + from routes_tables import ut_assembly + # ⛔ `consume_corrections=False`, and the default was a REAL BUG, not a tidy-up. + # `ut_assembly` defaults it True, so every `/notifications` read CONSUMED the + # one-shot field-name correction acks for every `ut_` topic that has an alert — + # taking them from the `/workspace` refresh that exists to show them to the person + # who made the edit. The customer branch below has always passed False; this one + # inherited a default nobody re-read. An inbox poll must never consume a one-shot. + g = ut_assembly(owner_sess, topic, + storage_key=f"{owner_sess.tenant}:{topic}:{owner_sess.uname}", + consume_corrections=False) + else: + from routes_customers import grid_assembly + g = grid_assembly(owner_sess, scope=topic, consume_corrections=False) + except Exception as e: # noqa: BLE001 + # ⭐ W32-T22 — SKIPPING IS FINE HERE; SKIPPING ANONYMOUSLY IS NOT. This one must not + # raise (one bad alert cannot empty an inbox), so unlike `_require_filtered_view` it + # keeps a blanket catch — but it now reports the refusal's OWN code where there is + # one. `type(e).__name__` said `HTTPException` for four different causes, and + # `lastError` is the only place a user ever learns why an alert stopped firing. + # + # ⚠ `with_rows=True` STAYS on this path, deliberately: unlike the create door, an + # evaluation genuinely needs the rows to run the filter over. So an alert on a + # read-through grid is created (T22) and then skips at evaluation with + # `window_required` naming why — which is D-184's remaining half, and it is a + # SENTENCE now rather than silence. + return {"skipped": _refusal_code(e) or "unavailable", "detail": type(e).__name__} + if isinstance(assemblies, dict): + assemblies[memo_key] = g + + view = _view_by_id(g, rec.get("viewId")) + if not isinstance(view, dict): + # Deleted, or un-shared out from under the alert. Say so on the RECORD rather than + # deleting the alert: an alert that silently vanishes is indistinguishable from one that + # never fires, and the user cannot debug what is not there. + return {"skipped": "view_missing"} + + # The SAME row build the grid and `/customers` use — `rows_from_pool` is what puts derived + # and overlay values on a row. Evaluating a filter against raw pool dicts would silently + # never match any condition on a user-created or measure column. + # + # ⛔⛔ AND "THE SAME ROW BUILD" WAS NOT TRUE, WHICH MADE EVERY ALERT ON A `ut_*` DATABASE BLIND + # TO IMPORTED DATA. Found by a verifier driving one real assembly through both paths. + # + # `routes_tables.table_rows` — the grid the person is looking at — merges the DEFINITION rows + # underneath the overlay ("base first, overlay wins"; that merge is itself the fix for owner + # item 3, *"it all got reseted"*). `_evaluate` is a second copy of that read and never got it: + # it handed `ws['overlays']` to `rows_from_pool` raw, so for a `ut_*` table every base cell + # evaluated as BLANK. Measured on one assembly, same view, same rows: + # rows_src state='unpaid' / 'paid' + # _evaluate saw state='' / '' ⇐ every base cell blank + # the GRID saw state='unpaid' / 'paid' + # so `state eq unpaid` matched NOTHING while the view showed one row, and `state isEmpty` + # matched EVERYTHING while the view showed none. **The alert did not merely miss rows — it + # inverted.** End to end: a row whose value arrived by import, automation, paste or the create + # door never fired; only a value typed as a hand EDIT did. + # + # ⚠ Scope, so nobody widens the fix past its cause: materialised `ut_*` tables are hit; + # `customer`/`product` are not (their fields are `source: "odoo"` and read off `rows_src`); + # `ut_odoo_*` never reaches here (`with_rows=True` refuses first and returns + # `skipped: window_required`). + # ⛔ ORDER IS LOAD-BEARING AND IS THE GRID'S: base underneath, overlay ON TOP. Inverting it + # would let a stale definition value shadow an edit the user has just made — the same defect + # `table_rows`' own note records, arriving from the other side. + _ov = (g.get("ws") or {}).get("overlays") or {} + _merged = {} + for _r in g["rows_src"]: + _pid = str(_r.get("pid")) + _cells = {k: v for k, v in _r.items() if k != "pid"} + _o = _ov.get(_pid) + if isinstance(_o, dict): + _cells.update(_o) + _merged[_pid] = _cells + rows = aios_grid.rows_from_pool(g["rows_src"], g["fields"], _merged, + derived=g.get("derived")) + config = view.get("config") or view + ctx = filter_eval.EvalCtx( + cohort_sets={str(k): {str(p) for p in (v.get("memberPids") or ())} + for k, v in (g.get("lists") or {}).items() if isinstance(v, dict)}, + measure_sets=g.get("measure_sets") or {}, + today=g.get("today")) + pids = filter_eval.visible_pids(config.get("filters") or [], rows, g["fields"], ctx, + member_pids=config.get("memberPids")) + labels = {str(r.get("pid")): str(r.get("name") or r.get("pid")) for r in rows} + return alerts.evaluate(rec.get("id"), [str(p) for p in pids], + labels=labels, partial=False, st=session.runtime) + + +@router.get("/alerts") +def list_alerts(session: Session = Depends(require_session)): + return {"alerts": alerts.list_alerts(user=session.uname, is_admin=session.admin, + st=session.runtime)} + + +@router.post("/alerts") +def create_alert(body: dict = Body(default=None), session: Session = Depends(require_session)): + body = body or {} + view_id = str(body.get("viewId") or "").strip() + if not view_id: + raise err(400, "bad_view", "an alert needs the id of the view it watches") + topic = _topic_or_400(body.get("topic")) + _require_filtered_view(session, topic, view_id) + import uuid + aid = f"al_{uuid.uuid4().hex[:12]}" + rec = alerts.create(aid, view_id=view_id, topic=topic, owner=session.uname, + label=body.get("label") or "", st=session.runtime) + # SEED IMMEDIATELY, so the alert starts from "everything currently matching is old news". + # Deferring this to the first write hook would mean the next edit announces the whole view. + outcome = _evaluate(session, rec) + return {"alert": {**rec, "seeded": True}, "first": outcome} + + +def _require_filtered_view(session: Session, topic: str, view_id: str): + """400 unless `view_id` exists on `topic` AND actually narrows something. + + ⛔ AN ALERT ON AN UNFILTERED VIEW IS SILENTLY INCAPABLE OF ALERTING, which is worse than one + that is refused. `filter_eval` treats an inactive tree as "no narrowing, every row shows" + (`visible_pids`'s own rule), so such an alert seeds with the entire table and can never see an + entrant again — there is nothing left to enter. The owner's words are *"when a Record gets + into that Filter's criteria"*: no criteria, no alert, and said at creation rather than + discovered by never being notified. + + `is_rule_active` is the SAME activeness predicate the engine and the column tints use — a + half-typed rule is not a filter, and this must agree with what actually narrows or it would + accept a view whose one rule the engine then ignores. + + ⭐⭐ WAVE 32 · T22 (owner item 17) — THIS FUNCTION WAS THE ERROR. Two defects, stacked, and + the second one hid the first. + + (1) **IT ASKED FOR EVERY ROW OF A TABLE IT NEVER LOOKS AT.** The only thing read below is + `g["views"]`. `ut_assembly` defaults `with_rows=True`, so creating an alert on a + read-through grid built the whole pool — and `scoped_pool` refuses that with + `409 window_required` over 963,783 rows, exactly as it is supposed to. `with_rows=False` + (W31-T20's flag, built for precisely this) answers the same question with `scoped_pids`, + runs the SAME `_defn_or_refuse` wall, and does not refuse. **That is D-184's create half, + closed** — an alert on a read-through grid can now be made at all. + (2) **A BLANKET `except Exception` TURNED EVERY NAMED REFUSAL INTO A 503.** `HTTPException` + is an `Exception`, so `404 unknown_table`, `403 forbidden`, `409 window_required` and + `503 store_not_ready` — four refusals that each say what is wrong — were all replaced by + *"the table is unavailable — try again in a moment"*. ⛔ AND THAT SENTENCE NEVER REACHED + A USER EITHER: `alertsApi.errorMessage` discards the text of any status ≥ 500 by design + (a 5xx body is the server's internals), substituting *"Something went wrong on our + side."* — which is the owner's screenshot, word for word. A knowable cause returned as a + 5xx is invisible by construction, so re-wording the 503 could never have fixed this. + ⚠ The except is narrowed, not deleted: an UNEXPECTED failure is still a 503, because that is + honest. What it may no longer do is catch a refusal that already knows its own name. + """ + from fastapi import HTTPException + + from harness import filter_eval + + try: + if topic.startswith("ut_"): + from routes_tables import ut_assembly + # ⚠ `consume_corrections=False` — the customer branch has always passed it and this + # one inherited a default nobody re-read. Creating an alert must not eat the one-shot + # field-name correction acks belonging to the `/workspace` refresh that exists to show + # them to the person who made the edit. Same defect `_evaluate`'s header records. + g = ut_assembly(session, topic, + storage_key=f"{session.tenant}:{topic}:{session.uname}", + consume_corrections=False, with_rows=False) + else: + from routes_customers import grid_assembly + g = grid_assembly(session, scope=topic, consume_corrections=False) + except HTTPException: + raise # it already names its own cause + except Exception as e: # noqa: BLE001 + # Genuinely unexpected. Still a 503, and now it carries the exception TYPE — without it, + # the one path that reaches this branch is also the one path with nothing to debug from. + raise err(503, "unavailable", + f"the table could not be read ({type(e).__name__}) — try again in a moment") + view = _view_by_id(g, view_id) + if not isinstance(view, dict): + raise err(404, "no_view", "that view does not exist on this table") + nodes, _conj = filter_eval.tree_parts((view.get("config") or view).get("filters") or []) + + # ⛔⛔ WAVE 33 · T29 — **CORRECTION: THE BLOCK BELOW IS TRUE ABOUT THE CODE AND FALSE ABOUT + # PRODUCTION, AND IT MUST BE READ SECOND.** It claims the missing-argument `TypeError` "IS + # owner item 17" — the owner's *"Something went wrong"*. It was not, and it could not have + # been: at `cbcf005`, the build the owner was using, the dict-read on `views` sat ~10 lines + # ABOVE this call and raised `AttributeError` on EVERY request, so the walk never reached the + # leaf and the arity bug was unreachable. `_view_by_id`'s own header records that fix. + # + # ⚠ WHY THE STALE PARAGRAPH STAYS RATHER THAN GETTING DELETED: the arity bug was real, the + # fix was right, and the three reasons it hid are the most transferable thing in this file. + # What was wrong is only its CLAIM TO BE THE CAUSE. Two comment blocks in one function each + # naming themselves as the origin of the same screenshot are mutually exclusive, and the next + # reader believes whichever they meet first — which is why this correction sits above rather + # than below. Caught by a verifier that read the SHIPPED file at the deployed commit instead + # of the working tree. [[grep-output-is-not-source]] + # + # ⛔⛔ WAVE 32 · T22 — **THE CALL BELOW WAS MISSING AN ARGUMENT** (and wave 32 believed, wrongly, + # that this was owner item 17 — see the correction directly above). + # + # `is_rule_active(rule, columns)` takes TWO parameters (`harness/filter_sql.py`; every other + # caller in the repo passes both). This one passed ONE, so the moment the walk reached a LEAF + # rule it raised `TypeError: is_rule_active() missing 1 required positional argument`. + # + # ⚠ READ WHAT THAT MEANS BEFORE FIXING ANYTHING ELSE: the walk only reaches a leaf when the + # view HAS a condition — and a view with a condition is the only kind an alert is allowed on. + # A view with no filters yields an empty `nodes`, so `_any_active` returns False without ever + # calling this, and the reader gets the honest 400 `no_filter`. **So the only path that + # worked was the refusal path: "Alert me about new records" had never once created an alert + # on a filtered view.** ⛔ And the raise lands OUTSIDE the `try` above, so it was not even the + # 503 — it was a bare FastAPI 500, which `alertsApi.errorMessage` renders as *"Something went + # wrong on our side. Try again in a moment."*, the owner's screenshot word for word. + # + # ⚠ THREE THINGS HID IT, and they are worth more than the fix. (1) Python does not check + # arity until the line RUNS, and this line runs only on the success path of a feature whose + # every test exercised its refusals. (2) The `no_filter` 400 above it is a real, correct, + # well-tested refusal, so the door looked alive. (3) `verify_alerts.py` asserts the refusal + # (`no_filter` reaches the user) and the transport — never a creation. A gate can be green, + # thorough and honest about everything except the one path the feature exists for. + # + # `_columns_map` is the DEFINITION of fields -> the membership set `is_rule_active` looks a + # column up in; building a second dict here would be a second answer to one question, which + # is this wave's other headline defect in a different file. Its leading underscore is a real + # smell and is BOOKED (PENDING, mailbox/C.md) rather than worked around. + columns = filter_eval._columns_map(g.get("fields") or []) + + def _any_active(ns): + for n in ns or (): + if isinstance(n, dict) and isinstance(n.get("children"), list): + if _any_active(n["children"]): + return True + elif filter_eval.is_rule_active(n, columns): + return True + return False + + if not _any_active(nodes): + raise err(400, "no_filter", + "this view has no active filter, so no record can ever ENTER it — add a " + "condition to the view first, then create the alert") + + +@router.delete("/alerts/{alert_id}") +def delete_alert(alert_id: str, session: Session = Depends(require_session)): + rec = next((r for r in alerts.list_alerts(st=session.runtime) + if str(r.get("id")) == str(alert_id)), None) + if rec is None: + raise err(404, "no_alert", "that alert does not exist") + if str(rec.get("owner")) != str(session.uname) and not session.admin: + raise err(403, "not_yours", "only the alert's owner (or an administrator) can delete it") + alerts.delete(alert_id, st=session.runtime) + return {"ok": True} + + +@router.post("/alerts/{alert_id}/run") +def run_alert(alert_id: str, session: Session = Depends(require_session)): + rec = next((r for r in alerts.list_alerts(user=session.uname, is_admin=session.admin, + st=session.runtime) + if str(r.get("id")) == str(alert_id)), None) + if rec is None: + raise err(404, "no_alert", "that alert does not exist") + return _evaluate(session, rec) + + +@router.get("/notifications") +def notifications(session: Session = Depends(require_session)): + """The inbox — RE-EVALUATED on read, which is a deliberate design choice. + + ⭐ A-S1-2 RESOLVED THE OTHER WAY, and the reason is structural rather than a shortcut. The + plan was a push hook: the automation engine calls `after_write` when it lands rows. But + `run_async` runs on a BACKGROUND THREAD with no `Session` in scope, and an alert must be + evaluated as its OWNER (see `_evaluate`) — so a push hook would have to mint a session inside + a worker thread from a tenant runtime, which is exactly the kind of ad-hoc identity + construction that leaks scope. + + Pulling on read has none of that: the caller IS a session, the assemblies are already + scope-cached, and the user cannot observe the difference — an inbox is only ever read by + someone opening it. The cost is that a notification is minted when you LOOK rather than when + the row landed, so the `at` stamp is detection time, not arrival time. + + `after_write` stays exported for the day the engine can hand over a real identity. + + ⭐⭐ W31-T24 — ONE ASSEMBLY PER (TOPIC, OWNER), NOT ONE PER ALERT. + ⛔ MEASURED FIRST, AND THE MEASUREMENT CORRECTS AN EARLIER READING OF IT. This route is + **20 ms in-process and 3,280 ms live** on tenant #0 — but tenant #0 has **ZERO alerts** + (censused 2026-08-12), so the 20 ms is an EMPTY LOOP and says nothing at all about what the + re-evaluation costs. The live 3,280 ms is the two store reads either side of that loop. So the + body below is not slow today; it is UNEXERCISED, and every alert a tenant creates adds a whole + grid assembly to an inbox poll. The memo turns O(alerts) into O(distinct topic × owner), which + is the difference between "fine" and "three seconds per alert" the day somebody uses the + feature. ⚠ Making the read cheap by evaluating LESS is the obvious wrong fix and is not what + this does: every alert is still evaluated, against the same rows, in the same order. + """ + assemblies = {} + for rec in alerts.list_alerts(user=session.uname, is_admin=False, st=session.runtime): + try: + _evaluate(session, rec, assemblies=assemblies) + except Exception: # noqa: BLE001 + continue # one bad alert must not empty the pane + # ⭐ W32-T20 (C3): every item leaves through `inbox_view`, so a notification queued before + # this wave carries a `target` too. See `notification_view`'s header for why it is derived. + return inbox_view(alerts.inbox(session.uname, st=session.runtime)) + + +@router.post("/notifications/read") +def read_notifications(body: dict = Body(default=None), + session: Session = Depends(require_session)): + body = body or {} + ids = body.get("ids") + if ids is not None and not isinstance(ids, list): + raise err(400, "bad_ids", "ids must be a list, or null to mark every notification") + # ⚠ THE SAME ENRICHMENT ON BOTH DOORS. `mark_read` returns a fresh inbox, and the Inbox + # module re-renders from it — an un-enriched answer here would strip `target` off every row + # the moment somebody marked one read, i.e. the feature would work until first use. + return inbox_view(alerts.mark_read(session.uname, ids, read=bool(body.get("read", True)), + st=session.runtime)) + + +def after_write(session: Session, topic_key: str): + """THE WRITE HOOK — call after a write that could change what a view matches. + + Exported as a plain function (not a route) so `core.grid_events`' callers and S2's automation + upserts reach it the same way. It never raises: an alert evaluation failing must not fail the + edit that triggered it. + + ⭐ W31-T24 — it shares `/notifications`' memo shape for the same reason: a write that changes + one view can trip several alerts on the SAME topic, and each would otherwise rebuild the table. + ⚠ STILL ZERO PRODUCTION CALLERS (W31-T24 confirmed it; the route docstring above says why the + push hook was resolved the other way). Booked rather than wired: minting a session inside the + engine's worker thread is the ad-hoc identity construction this file exists to avoid. + """ + try: + assemblies = {} + return alerts.after_write(topic_key, st=session.runtime, + runner=lambda rec: _evaluate(session, rec, + assemblies=assemblies)) + except Exception: # noqa: BLE001 + return {"evaluated": 0} diff --git a/api/routes_automation.py b/api/routes_automation.py index b2e7d11bafc1f0a8f5d0e53caabcd7dbb820bd10..88849c492b7823d3b84c96c4db87725749c92132 100644 --- a/api/routes_automation.py +++ b/api/routes_automation.py @@ -1,1896 +1,1896 @@ -"""routes_automation.py — wave-18 item 5 (contract C4-AUTO): the automation surface's API. - -The router is thin on purpose: everything that can be wrong about an automation — a malformed -cron, a URL the SSRF rail refuses, a key field that is not one of the mapped columns — is decided -in `automation_engine`, which is a pure-ish module a gate can drive without a server. This file -does auth, shape and status codes. - -⛔ THE TICK ENDPOINT IS THE ONE UNAUTHENTICATED ROUTE, AND IT IS FAIL-CLOSED TWICE OVER. It takes -no session (an external cron has no cookie), so it is gated on a shared secret in -`X-AIOS-TICK-TOKEN`; and when `AIOS_AUTOMATION_TICK_TOKEN` is UNSET the route refuses everything -rather than admitting everyone. A "no token configured means no check" default is how an internal -trigger becomes a public one — the same class of mistake as an empty-200 permission answer. -""" -import os -import re -import time -from datetime import datetime, timezone - -from fastapi import APIRouter, Body, Depends, Header, Request - -import ai_review -import automation_engine as engine -import oauth_connect -import routes_oauth -import scope_cache -from deps import Session, err, module_gate, require_session - -router = APIRouter(prefix="/api/v1") - -# C5 (wave 22): the OAuth connector surface rides INSIDE this router — main.py belongs to no -# session this wave, and this router is already mounted there. `/api/v1` + `/oauth/...`. -router.include_router(routes_oauth.router) -# ⛔⛔ D-203 — THE NESTED INCLUDE OF `routes_connectors` IS REMOVED, and the reason it existed is -# worth keeping because it was a GOOD reason that expired. -# -# Wave 23 (C11) mounted the connectors directory here rather than in `main.py`, because this router -# was already mounted and `main.py` belonged to another session — a sensible way to avoid a -# cross-fence ask. `main.py:227` includes `routes_connectors` DIRECTLY now, so this line stopped -# being the only door and became a SECOND one. -# ⚠ AND A ROUTER MOUNTED TWICE DOES NOT SERVE THE SAME PATHS TWICE — it serves the prefix twice. -# `routes_connectors` carries its own `/api/v1`, so nesting it inside this router's `/api/v1` -# produced **`/api/v1/api/v1/connectors/directory`**: a live, session-gated, entirely dead path -# that nothing links to and every route audit has to explain. MEASURED before removal: 123 served -# paths, exactly 1 of them doubled. -# ⚠ `routes_oauth` above is NOT the same case and stays: `main.py` does not mount it, so this -# router is genuinely its only door. Deleting it because its neighbour was wrong is how a real -# route dies for a tidy-up. -import routes_connectors # noqa: E402,F401 - -#: The registry key this surface carries (C-AUTONAV — A adds the row; the gate is live now, so -#: the day the row lands the wall is already the one that was tested). -MODULE = "automation" - -_GATE = module_gate(MODULE) - - -def _wire(defn, tenant, rt=None): - """One automation, as the client reads it. `running` is PROCESS state, never store state — - see the engine header on why a persisted 'running' is a permanent lock.""" - live = engine.running(tenant, defn.get("id")) - sched = defn.get("schedule") or {} - nxt = engine.next_fire(sched.get("cron")) if sched.get("enabled") else None - status = dict(defn.get("status") or {}) - if live: - status = {**status, "state": "running", "startedAt": live.get("startedAt"), - "step": live.get("step")} - return { - "id": defn.get("id"), "name": defn.get("name"), "kind": defn.get("kind"), - "config": defn.get("config") or {}, "schedule": sched, "status": status, - "runs": list(defn.get("runs") or [])[:engine.MAX_RUNS], - "created": defn.get("created"), "createdBy": defn.get("createdBy"), - "nextRunAt": nxt.strftime("%Y-%m-%d %H:%M") if nxt else "", - "running": bool(live), - # ⭐ WAVE 27 (contract C6) — DEBT D-70: "a paid search is already outstanding at the - # vendor". THE MONEY GUARD, and it was INERT for a whole wave because this line was - # missing: the client declared `awaitingResults` REQUIRED and read it in `runBlock()`, - # nothing ever sent it, so `undefined` was falsy and the Run button stayed armed. - # - # ⛔ WHY IT IS NOT `running`. `running` above is PROCESS state (`engine.running`) and is - # FALSE for the entire 20–30 MINUTE vendor wait, which IS the hazard window: the ticks - # are 3 minutes apart, so between them the automation is idle and the button re-arms. - # MEASURED the expensive way on 2026-08-06 (~$0.025): with no snapshot outstanding a - # Run-now press returned 200 and started a BRAND-NEW billable corpus search. - # - # ⛔ AND NOT A PERSISTED "running" FLAG EITHER — the engine header forbids one (it - # outlives the process and locks the automation forever). `state.pendingSnapshot` is the - # field that already exists, is already persisted, is already carried untouched through - # `clean_definition`, and is the SAME expression `pending_collect_ids()` uses to decide - # what to collect. One truth, two readers. - "awaitingResults": bool(str((defn.get("state") or {}).get("pendingSnapshot") - or "").strip()), - # C3 (wave 22): the trigger config rides whole — the webhook token included, because - # the person configuring the external caller has to be shown the URL somewhere, and - # this payload is session-gated behind the same wall as everything else here. - # ⭐⭐ WAVE 35 · T36 / R10 — THE PARKED BATCH, SUMMARISED. Without this line the whole - # ticket would be unreachable: `_wire` is a WHITELIST, so a key the engine parks on the - # definition simply never arrives, and the detail pane could not know a batch was waiting - # ([[reachable-is-not-the-same-as-built]] — the exact shape `flow` was caught in above). - # ⚠ COUNT AND NOTES ONLY, NEVER `items`. A 200-statement batch carries 200 rendered HTML - # mails; this payload rides on the automations LIST, which every rail paint reads. - # `GET /admin/statements/agent/{id}` serves the detail when somebody opens the batch. - "pendingStatements": ({ - "count": int((defn.get("pendingStatements") or {}).get("count") or 0), - "ts": (defn.get("pendingStatements") or {}).get("ts") or "", - "notes": list((defn.get("pendingStatements") or {}).get("notes") or [])[:25], - } if isinstance(defn.get("pendingStatements"), dict) else None), - # ⭐⭐ WAVE 35 · T37 — `system` FOR A **STORED** ROW. Until now only SYNTHETIC rows carried - # it, stamped by their builders AFTER this function (`_odoo_sync_row`, `_field_agent_rows`), - # so a stored row with the marker had it enforced on the server (`delete_automation` 409s) - # and INVISIBLE to the client — which paints a live-looking Delete that answers with a - # refusal. `AutomationDetail` already reads `automation.system` to disable that button and - # explain why; this is the line that lets it. - # ⚠ Absent stays absent rather than becoming `""`: every ordinary automation is not-a-system - # -agent, and an empty string is a value a client could accidentally treat as one. - **({"system": str(defn.get("system") or "")} if defn.get("system") else {}), - "trigger": defn.get("trigger") or None, - "statusNote": defn.get("statusNote") or "", - # The one-sentence summary (airtable-brief rec 7), composed from the definition so it - # cannot describe steps the engine does not run. - "sentence": engine.compose_sentence(defn), - # THE CANVAS TOPOLOGY (R5) rides with the automation rather than being rebuilt in the - # client, for the same reason `cronPresets` does: the engine that RUNS the steps is the - # only thing entitled to say what the steps are. - "graph": engine.graph(defn), - # ⭐ WAVE 23 (C4/C5) — THE BUILDER'S OWN STATE, and its absence here was a silent-drop - # bug caught in review rather than by a gate: `flow` was stored, patchable and validated, - # but never sent back. B would have saved a flow through PATCH, got a 200, and watched - # every action vanish on reload — the classic "it didn't save" with nothing red anywhere. - "flow": defn.get("flow") or {"actions": []}, - # ⭐⭐ WAVE 32 · T45 (owner item 10) — CONFIGURED / UNCONFIGURED, per action, on the wire. - # - # ⛔ THE SERVER SAYS IT, not the client, and that is the whole reason it is here: the same - # `engine.action_needs` answers this label, `engine.run_refusal`'s 400 and `run_now`'s own - # refusal, so a card cannot read "Configured" over an action the run will refuse. Three - # readers, one predicate — the alternative is a client-side rule that agrees with the - # server until somebody adds a key to one of them (`awaitingResults` above is this file's - # own record of what the other shape costs). - # ⚠ IT IS A LIST, KEYED BY ACTION ID, and it names the NESTED actions too — an unconfigured - # step inside an If / then branch is exactly the one a person cannot see. - # ⚠ COSTS NO STORE READ. `action_needs` is pure over `(kind, config)`; this route is the - # one W30-T12 took the `user_tables` deep copy off, and a label is not worth putting it - # back. What needs the target database's schema (an enrich binding resolved by the profile - # FLAG) stays a run-time refusal — see `ACTION_REQUIRED`'s note. - "unconfigured": engine.unconfigured_actions(defn), - # ⭐⭐ WAVE 36 · W36-T38 (owner item 8 / R9) — WHICH ACTIONS AN AGENT BUILT. - # `{action_id: {agent, agentName, created, by}}`, empty for an ordinary automation. - # ⛔ THE CONFIG IS NOT HIDDEN AND MUST NOT BE. R9 is a WRITE rule: the client uses this to - # draw a lock and explain who owns the step, never to withhold what the step does — an - # automation nobody can audit is worse than one nobody can edit. - # ⚠ `rt=None` YIELDS `{}` RATHER THAN OMITTING THE KEY, so a caller that forgets the - # runtime produces "nothing is agent-owned" (a visible, wrong-but-safe answer) instead of - # an absent prop the client silently reads as undefined - # [[flag-shipped-without-its-writer]]. - "agentActions": (_agent_marks(rt).get(str(defn.get("id") or "")) or {}) if rt else {}, - } - - -#: ⭐⭐ WAVE 34 · CONTRACTS C3 + C4 — WHY A ROW CANNOT BE DELETED, as a slug. -#: -#: ⚠ ALWAYS A STRING, NEVER A BOOLEAN, and the two contracts disagreed about that (C3 says -#: `system: true`, C4 says `system: ""`). Raised as ask `E-13`; built as a slug because one -#: key with two types is [[one-question-two-normalizers]] before a line is written, and because a -#: refusal has to be able to NAME the reason. Truthiness is unchanged for any reader that only -#: asks "is this a system agent". -SYSTEM_FIELD_AGENT = "field_agent" -SYSTEM_ODOO_SYNC = "odoo_sync" - - -def _field_agent_rows(session): - """⭐⭐ CONTRACT C3 — one synthetic Agent row per `ai_enrich` column in the tenant. - - R13: *"Every field agent must be VIEWABLE under the Agent module as a simple Trigger whose - first step is the field enrichment, with the Field and Database shown under Config."* - - ⛔⛔ THESE ARE DERIVED, NOT STORED, AND NOTHING HERE MAY WRITE. A field agent's only source of - truth is the column definition F's `_clean_field` stamps; a copy in the automations bucket - would be a second one, and the first PATCH through the enrichment editor would silently - diverge from it. `delete_automation` refuses these ids for the same reason: accepting a delete - that cannot delete anything is the `view_upsert` failure mode (200 OK, nothing written) in a - new place. - - ⛔⛔ AND IT USES THE ROWS-FREE PROJECTION, WHICH IS THE WHOLE ENGINEERING PROBLEM OF THIS - TICKET. `GET /automations` is the route W29-T01 and W30-T12 spent two waves taking the - whole-document read OFF: it reads `user_tables` exactly ONCE PER TENANT PER PROCESS now, and - this file's own `unconfigured` note says *"a label is not worth putting it back"*. Walking - every table's fields for `ai_enrich` columns is exactly that read. So this uses - `user_tables.all_defs` — the same projection `/nav` uses, MEASURED there at 1,750 ms -> 1.4 ms, - because tenant #0's document is 99.89% rows and a field definition is not a row. - ⚠ `all_defs` RAISES on `rows` rather than answering `{}`. Nothing here wants a row; if that - ever changes, the error names the projection instead of painting an empty grid. - - ⚠ THE PERMISSION WALL IS CALLED, NEVER RE-IMPLEMENTED. `may_open` decides which databases this - caller may see, lent the projected document exactly as `/nav` lends it. This route's own - `automation_tables` carries the scar that makes that non-negotiable: it once re-implemented - the wall, got it WIDER, and showed people databases they were refused the moment they clicked. - """ - import core.user_tables as user_tables - - try: - defs = user_tables.all_defs(st=session.runtime) or {} - lent = user_tables.lend(session.runtime, **{user_tables.STORE_KEY: defs}) - except Exception: # noqa: BLE001 - # A store blip must not take the whole rail down with it. The stored automations are the - # payload's real subject; field agents are an addition to it. - return [] - - out = [] - for key, defn in sorted(defs.items(), key=lambda kv: (kv[1].get("label") or "").lower()): - if not user_tables.may_open(key, session.uname, session.admin, st=lent): - continue - table_label = defn.get("label") or key - for f in user_tables.ai_enrich_fields(defn): - bag = f.get("automation") if isinstance(f.get("automation"), dict) else {} - if bag.get("kind") != SYSTEM_FIELD_AGENT: - continue - col = str(f.get("key") or "") - if not col: - continue - # ⚠ THE ID CARRIES THE TABLE AND THE COLUMN, and `table` is NOT read off the bag: - # F's `_clean_field` is handed ONE field dict with no table context, so it cannot - # stamp one (mailbox `F-3`). E holds that key because E is the side that iterates - # tables. Two sources for one fact was the alternative, and it drifts. - trig = bag.get("trigger") if isinstance(bag.get("trigger"), dict) else {} - mode = str(trig.get("mode") or "manual") - cron = str(trig.get("cron") or "").strip() - defn_syn = { - "id": f"field:{key}:{col}", - "name": f.get("label") or col, - "kind": SYSTEM_FIELD_AGENT, - # ⛔⛔ TWO TRIGGER VOCABULARIES MEET HERE AND THEY ARE NOT THE SAME ONE. A field - # agent's trigger is the ENRICHMENT vocabulary (`{mode, cron}`, values - # manual/on_change/schedule); an automation's is the ENGINE's (`{key}` plus a - # separate `schedule {cron, enabled}`). Handing the engine's shape a `mode` it - # cannot read is not a type error — it is SILENT: `compose_sentence` fell back to - # "When you press Run now" for a column scheduled at 06:00, and `graph()` drew a - # trigger node with no trigger. Measured before this mapping existed, which is the - # only reason it does. - # ⚠ THE MAP IS EXACT WHERE AN EQUIVALENT EXISTS AND HONEST WHERE IT DOES NOT: - # `manual` and `schedule` are the engine's own keys, and `on_change` becomes - # `event_field` ("When a record matches conditions"), the nearest thing the engine - # has to "a cell this column reads has moved". `mode` is kept verbatim beside it so - # nothing is lost in translation and the enrichment editor stays the source. - "trigger": {"key": {"manual": "manual", "schedule": "schedule", - "on_change": "event_field"}.get(mode, "manual"), - "mode": mode, **({"cron": cron} if cron else {})}, - **({"schedule": {"cron": cron, "enabled": True}} - if mode == "schedule" and cron else {}), - "flow": {"actions": [{"id": "act_1", "kind": "ai_enrich", - "config": {"table": key, "field": col, - "tableLabel": table_label}}]}, - } - # ⭐ THROUGH `_wire`, NOT BESIDE IT. A synthetic row hand-built to "look like" a - # stored one is a second shape that agrees until somebody adds a key to the real one - # — the exact way `awaitingResults` shipped inert for a whole wave. Passing the - # synthetic DEFINITION through the same function makes them identical by - # construction; only `system` is stamped afterwards, because no stored row has it. - row = _wire(defn_syn, session.tenant, rt=session.runtime) - row["system"] = SYSTEM_FIELD_AGENT - out.append(row) - return out - - -#: ⭐⭐ WAVE 34 · W34-T47 (mailbox D-5) — R6 APPLIED TO WORDS A MODEL WROTE. -#: -#: ⛔ `web_prose` READS FILES, so it is structurally blind to a dash that arrives at RUN TIME. -#: `POST /automations/draft` returns three strings a language model authored (the flow's name, -#: each step's `why`, and the dropped/notes sentences) and `W34-T42` is what put them on a screen. -#: Lane D MEASURED that a prompt instruction does not hold: their system prompt ends "Never use an -#: em dash" and the very next live cerebras turn came back with one. So this is code at the -#: boundary, not a better prompt. -#: ⚠ A DIGIT RANGE IS A DIFFERENT SENTENCE and gets the first rule: "10-20" means "10 to 20", and -#: turning it into "10, 20" states two numbers where the model stated a span. -_DRAFT_DASH = "[" + chr(0x2014) + chr(0x2013) + "]" - - -def _draft_no_dashes(text): - """Model prose with no em or en dash, and no meaning changed on the way.""" - text = str(text or "") - text = re.sub(rf"(?<=\d)\s*{_DRAFT_DASH}\s*(?=\d)", " to ", text) - text = re.sub(rf"\s*{_DRAFT_DASH}\s*(?=[,.;:!?])", "", text) - text = re.sub(rf"(?<=[,;:])\s*{_DRAFT_DASH}\s*", " ", text) - return re.sub(rf"\s*{_DRAFT_DASH}\s*", ", ", text) - - -ODOO_SYNC_ID = "system:odoo_sync" -#: The cadence presets in the words a person reads. ⚠ Keyed on `odoo_relational.SYNC_PRESETS`, and -#: a key with no phrase here falls back to "Every " rather than raising — a missing caption -#: must degrade to something readable, not take the agents list down. -ODOO_CADENCE_PHRASE = {"30m": "Every 30 minutes", "1h": "Every hour", - "4h": "Every 4 hours", "daily": "Every day"} - - -def _odoo_sync_row(session, detail=False): - """⭐⭐ CONTRACT C4 (ruling R22) — the Odoo sync, as an agent you can SEE. - - R22: *"Implicit time-triggered work (Odoo syncing) becomes EXPLICIT: a pre-set agent that - cannot be deleted, whose Canvas shows Trigger = time and Action = data syncing from Odoo, with - the keychain/API configuration shown."* - - ⛔⛔ DERIVED, NOT STORED, AND C4 SAYS "a stored automation may carry `system:`". THE MECHANISM - C4 ASKS FOR IS BUILT (`delete_automation` refuses any stored row carrying `system`); this - particular agent does not use it, for three measured reasons, and the deviation is raised as - ask `E-15` rather than taken quietly: - 1. NOTHING CAN SEED IT. There is no system-automation seeding path anywhere, and `D-195` - measured three times that a CLI write to the tenant store while the Space is live REPORTS - SUCCESS and is reverted within a minute. A stored row would have to be minted by the - container, i.e. a write-on-read on the busiest list route in the product. - 2. STORED MEANS RUNNABLE BY MACHINERY BUILT FOR USER AUTOMATIONS. If its runs went through - `_commit_run`, `CONSECUTIVE_FAILURE_PAUSE` would flip `schedule.enabled` off after K bad - passes — silently disabling the tenant's REAL Odoo cadence, not a cosmetic card. - 3. TWO COPIES OF ONE CADENCE. The loop reads `odoo_relational.sync_seconds`; a stored - automation would carry its own `schedule.cron`, and C4's own requirement is that the card - "cannot drift from the running loop". Deriving makes drift impossible instead of - forbidden. - - ⚠ `detail=False` IS THE LIST PATH AND IT STAYS CHEAP. The keychain label and the last-sync - stamp each cost their own store read, and `GET /automations` is the route two waves were spent - taking reads OFF. They are resolved only on the DETAIL fetch, which is one automation and one - click. Returns `None` when this tenant has no Odoo at all — an agent for a connector nobody - connected is a card that lies. - """ - try: - import odoo_relational as rel - except Exception: # noqa: BLE001 - return None - try: - cfg = rel.read_config(session.runtime) - secs = rel.sync_seconds(session.runtime) - frozen = bool(rel.frozen(session.runtime)) - except Exception: # noqa: BLE001 - return None - - every = str(cfg.get("syncEvery") or rel.DEFAULT_SYNC) - manual = secs is None - defn = { - "id": ODOO_SYNC_ID, - "name": "Odoo sync", - "kind": SYSTEM_ODOO_SYNC, - # ⚠ THE CADENCE IS READ, NEVER STORED HERE. `everySeconds` is what the loop will actually - # sleep for, so the card cannot claim a schedule the loop is not keeping. - "trigger": {"key": "manual" if manual else "schedule", "preset": every, - "everySeconds": secs, "presets": sorted(rel.SYNC_PRESETS)}, - **({} if manual else {"schedule": {"cron": "", "enabled": True}}), - "flow": {"actions": [{"id": "act_1", "kind": "odoo_sync", - "config": {"connector": "odoo", "every": every, - "frozen": frozen}}]}, - } - row = _wire(defn, session.tenant, rt=session.runtime) - row["system"] = SYSTEM_ODOO_SYNC - # ⛔ THE SENTENCE IS OVERRIDDEN, AND IT IS A FIX RATHER THAN A PREFERENCE. `compose_sentence` - # speaks the automation vocabulary — it builds "When , run N actions" out of - # `schedule.cron` — and this agent has no cron: its cadence is a PRESET in seconds that the - # resync loop sleeps on. MEASURED before this line existed, the card read exactly - # `", run 1 action."`: a leading comma where the trigger phrase should have been. Two - # vocabularies again, and this one fails in punctuation rather than in behaviour, which is - # why only reading the output catches it. - # ⚠ AND THE CADENCE IS SPELLED OUT RATHER THAN INTERPOLATED. `f"Every {every}"` produced - # "Every daily, sync data from Odoo." — the preset KEYS are storage tokens ("30m", "daily"), - # not English, and a sentence built by pasting one in is only accidentally readable for the - # three that happen to be durations. Caught by reading the output, not by any assertion. - row["sentence"] = ("Odoo is disconnected, so nothing syncs." if frozen else - "Manually, sync data from Odoo." if manual else - f"{ODOO_CADENCE_PHRASE.get(every, f'Every {every}')}, sync data from Odoo.") - # ⛔ R6's SECOND SENTENCE, WHICH IS THE HALF THAT GETS DROPPED: a limit that cannot be removed - # must be REPORTED with its cause. Two are reported here rather than left for somebody to - # discover by watching a mirror not move. - notes = [] - if frozen: - notes.append("Odoo is disconnected, so this agent is not syncing. Reconnect it in " - "Connectors.") - if manual: - notes.append("The cadence is set to manual, so nothing syncs on a timer.") - row["statusNote"] = " ".join(notes) - if not detail: - return row - - # ── the DETAIL half: two store reads, on a route that fetches one automation ────────────── - try: - import routes_keychain as kc_routes # noqa: PLC0415 - row["flow"]["actions"][0]["config"]["keychain"] = ( - kc_routes.odoo_config(session).get("label") or "Odoo") - except Exception: # noqa: BLE001 - # A label is a nicety; failing to read one must not make the agent unopenable. - row["flow"]["actions"][0]["config"]["keychain"] = "Odoo" - try: - from harness import datastore as ds # noqa: PLC0415 - # ⭐ THE LAST RUN IS THE MIRROR'S OWN `_sync_state`, not a run history we would have to - # write. That table already carries a real per-entity `updated` stamp, so the card reports - # what actually happened rather than what this module remembers happening. - st = ds.status() or {} - stamps = sorted(str((v or {}).get("updated") or "") for v in st.values() if v) - last = [s for s in stamps if s] - if last: - row["status"] = {**(row.get("status") or {}), "state": "ok", "lastRunAt": last[-1], - "lastSummary": f"{len(last)} datasets synced"} - except Exception: # noqa: BLE001 - pass - return row - - -def _patch_odoo_sync(session, body): - """⭐⭐ CONTRACT C4 — THE ONE THING ON THIS CARD THAT IS EDITABLE: how often it runs. - - C4: *"Its Canvas is READ-ONLY except `trigger.config.everySeconds`, which writes through to the - same value `odoo_relational.sync_seconds` reads, so the card cannot drift from the running - loop."* - - ⛔ IT WRITES THE CONNECTOR'S OWN CONFIG KEY, NOT AN AUTOMATION. `rel.CONFIG_KEY` / - `rel.SYNC_PRESETS` are the same key and the same vocabulary `sync_seconds` reads and - `_store_resync_loop` sleeps on, so there is one value and the card reads back exactly what the - loop will use. Storing a cron on a synthetic automation would have been a second copy with a - guaranteed drift date. - - ⚠ EVERY OTHER FIELD IS REFUSED RATHER THAN IGNORED. A PATCH that silently kept only the part - it liked would let somebody rename this agent, watch the 200, and find the name gone on reload - — the failure shape this module has already paid for twice. - - ⚠ AND THE LATENCY IS REPORTED, because it is real and a person would otherwise call it a bug: - the loop re-reads the cadence at the TOP of its cycle and then sleeps, so a change takes effect - on the NEXT pass — up to one full OLD interval away (24 hours if it was on `daily`). - """ - import odoo_relational as rel # noqa: PLC0415 - - known = {"trigger", "every", "syncEvery"} - extra = sorted(k for k in body if k not in known) - if extra: - raise err(409, "system_agent", - "this agent is the Odoo connection's own sync. Only how often it runs can be " - f"changed here; {', '.join(extra)} belongs to the connector in Connectors") - every = str(body.get("syncEvery") or body.get("every") - or ((body.get("trigger") or {}) if isinstance(body.get("trigger"), dict) else {}) - .get("preset") or "").strip().lower() - if every not in rel.SYNC_PRESETS: - raise err(400, "bad_cadence", - f"pick one of: {', '.join(sorted(rel.SYNC_PRESETS))}") - - def _up(cur): - cur = dict(cur) if isinstance(cur, dict) else {} - cur["syncEvery"] = every - return cur - - session.runtime.update(rel.CONFIG_KEY, _up, flush="sync") - row = _odoo_sync_row(session, detail=True) - if row is None: - raise err(404, "unknown_automation", "no automation with that id") - secs = row["trigger"].get("everySeconds") - note = ("It syncs on a timer again from the next cycle." if secs - else "Nothing will sync on a timer now.") - return {"automation": row, - "notes": [f"The Odoo sync is set to {every}. {note} A change takes effect on the " - f"loop's next pass, so it can be up to one of the OLD intervals away."]} - - -def _triggers_vocab(session): - """C3's server-owned trigger list: `[{key, label, ready, needs, planned}]`, keys EXACTLY - `engine.TRIGGER_KEYS` + `engine.TRIGGER_PLANNED`. `ready:false` + `needs` renders as a - not-configured state — never a dead control, never a client-side union. - - ⭐ WAVE 23 (R2): the list is the WHOLE Airtable-parity vocabulary, and the two triggers we - have not built ride it with `planned: true`. That is the honest version of "show all, wire - eight": the picker paints them faded with a reason instead of a shorter list that quietly - implies the missing ones do not exist. `clean_trigger` refuses them, so the faded state is - enforced at the door and not merely in the client's `disabled` attribute. - """ - tick_on = _tick_state()["enabled"] - g = oauth_connect.status(session.runtime, session.uname).get("google") or {} - email_ready = bool(g.get("configured")) and bool(g.get("connected")) \ - and not g.get("reconnect") - email_needs = "" if email_ready else ( - "connect_gmail" if g.get("configured") else "configure_google") - per = { - "manual": (True, ""), - "schedule": (tick_on, "" if tick_on else "arm_tick"), - "event_field": (True, ""), - "record_updated": (True, ""), - "record_created": (True, ""), - "enters_view": (True, ""), - "webhook": (True, ""), - "email": (email_ready, email_needs), - "form_submitted": (True, ""), - # ⭐ WAVE 24 (C-TRIG) — Instagram discovery, now a trigger. Readiness is the vendor key, - # the same bit `paidReady` carries: with no key the search door is closed and the picker - # must say so rather than offering a control that silently finds nothing. `clean_trigger` - # still ACCEPTS it either way — readiness is a deployment fact, not a validity one, which - # is the same split `email` already makes. - "ig_profile_match": (engine.bd_ready(), "" if engine.bd_ready() else "configure_brightdata"), - # ⭐ WAVE 29 (D-9 / R1) — TikTok, live. ⛔ IT NEEDS ITS OWN ROW EVEN THOUGH THE ANSWER IS - # IDENTICAL, and the reason is the `.get(k, (True, ""))` default below: a trigger this dict - # forgets is reported READY, so a deployment with no vendor key would offer TikTok search - # as configured and the search would find nothing. Same key, same readiness bit, stated. - "tiktok_profile_match": (engine.bd_ready(), - "" if engine.bd_ready() else "configure_brightdata"), - } - #: ⭐ WAVE 24 — the server's own one-line description per trigger. C-TYPES: the picker renders - #: THIS under the option, because a CLIENT paraphrase of a server vocabulary is a second copy - #: of it, free to drift. Absent = the client shows nothing, never something invented. - detail = { - "manual": "It runs only when you press Run now", - "schedule": "It runs on a repeating schedule", - "event_field": "A record in the database starts matching a condition you set", - "record_updated": "Any of the columns you watch is changed", - "record_created": "A new record is added to the database", - "enters_view": "A record starts appearing in a saved view", - "webhook": "Something outside calls this automation's URL", - "email": "A message arrives in the connected mailbox", - "form_submitted": "Somebody submits one of this database's forms", - "ig_profile_match": "Search Instagram for profiles matching your filters, on a schedule", - "button_clicked": "Somebody presses a button on a record", - "comment_added": "Somebody comments on a record", - "web_page_changed": "A page you are watching is different from last time", - "tiktok_profile_match": "Search TikTok for profiles matching your filters, on a schedule", - } - - def _taxonomy(k): - """⭐ WAVE 25 · C2 — the four taxonomy keys, composed from the ENGINE's maps. - - ⛔ `.get(k) or FALLBACK`, NEVER `TRIGGER_GROUP_OF[k]`. The first draft of this indexed the - map on the reasoning that a default is how a Connector trigger quietly appears under - Database — and the gate rejected it, correctly, against the incident `per.get(k, ...)` - eight lines below records: a key added to `TRIGGER_KEYS` without remembering a dict beside - it raised KeyError and took `GET /automations` down, i.e. the whole surface, which polls - this every 2.5 s. A mis-grouped row is cosmetic; a 500 is not, and the ranking is not - close. - ⚠ THE CLASSIFICATION IS STILL MANDATORY — it is enforced at the GATE (no shipped trigger - may land in `other`) rather than at the request. Soft here, hard there. - """ - g = engine.TRIGGER_GROUP_OF.get(k) or engine.TRIGGER_GROUP_FALLBACK - return {"group": g, - "groupLabel": engine.TRIGGER_GROUPS[g]["label"], - "groupOrder": engine.TRIGGER_GROUPS[g]["order"], - # The SUB-group inside "Connector"; None everywhere else. ⚠ Group by this key, - # render its label — it is NOT a connector-directory slug (see the engine's note). - "connector": engine.TRIGGER_CONNECTOR.get(k), - # D-55: "the cron drives this one". Derived from the engine's schedule set MINUS - # `manual`, so the client's `CRON_DRIVEN_TRIGGERS` copy can be deleted. - "schedules": k in engine.TRIGGER_CRON_KEYS} - - out = [] - for k in engine.TRIGGER_KEYS: - # ⚠ `.get` WITH A DEFAULT, NOT `per[k]`. This loop walks the ENGINE's vocabulary and - # indexed a hand-maintained dict beside it: adding a key to `TRIGGER_KEYS` without - # remembering this dict raised KeyError and 500'd `GET /automations` — the payload the - # whole automation surface polls every 2.5 s — with every gate and `tsc` still green. - # Defaulting to "ready, needs nothing" is the honest fallback: a trigger the engine - # offers and this route has no readiness opinion about is simply available. - ready, needs = per.get(k, (True, "")) - row = {"key": k, "label": engine.TRIGGER_LABELS[k], "ready": ready, "needs": needs, - "planned": False, "detail": detail.get(k, ""), - # A3(3): the connect affordance is SERVER-COMPOSED — the client never maps a - # `needs` token to a route, so B's CONNECT_PROVIDERS shim deletes itself. - "connect": None, **_taxonomy(k)} - if not ready and needs == "connect_gmail": - row["connect"] = {"provider": "google", - "startUrl": "/api/v1/oauth/google/start"} - out.append(row) - for k in engine.TRIGGER_PLANNED: - out.append({"key": k, "label": engine.TRIGGER_LABELS[k], "ready": False, - "needs": "coming_soon", "planned": True, "connect": None, - "detail": detail.get(k, ""), **_taxonomy(k)}) - return out - - -def _tick_state(): - """⭐ WAVE 21 (C6 amendment A1) — can a SCHEDULE fire on this deployment? - - TWO independent paths can: the in-process scheduler (`AIOS_AUTOMATIONS=1`, - `automation_engine.py` module bottom) and an external cron POSTing `/automations/tick`, - gated on `AIOS_AUTOMATION_TICK_TOKEN` (AWS EventBridge in production). The Step-1 Trigger - card must be honest in both directions: "schedules won't fire" on a deployment where - EventBridge demonstrably fires them daily is the exact lie R9 forbids. `external` means - "the door is OPEN", never "the caller is alive" — the client's copy says so.""" - inproc = os.environ.get("AIOS_AUTOMATIONS") == "1" - ext = bool(os.environ.get("AIOS_AUTOMATION_TICK_TOKEN")) - return {"enabled": bool(inproc or ext), - "source": "in-process" if inproc else ("external" if ext else "")} - - -#: W29-T01 — the Board retirement ran, per tenant, this process. Same shape and same reasoning as -#: `_C8_MIGRATED` below: a one-way cleanup of data no living code path creates any more, whose -#: cost is a full walk of every row-cell in the tenant and whose result on pass 2..N is always -#: "nothing to do". ⚠ A module-global keyed by tenant is deliberately NOT reset by a table create -#: or delete — a new database cannot contain the legacy stage cells this retires. -_BOARD_RETIRED = set() - -#: ⭐ WAVE 35 · T37 — tenants whose statements agent has been considered THIS PROCESS. Same shape as -#: `_BOARD_RETIRED` and, like it, purely a cost saver: the DURABLE idempotency is the row's own -#: existence, which `engine.seed_statements_agent` re-checks inside its own store mutation. -#: ⚠ SO A RESTART RE-CHECKING IS HARMLESS BY CONSTRUCTION, which is the property to preserve. If -#: this set were ever the only guard, a process restart would mint a second agent. -_STATEMENTS_SEEDED = set() - -#: ⭐⭐ WAVE 30 · T12 — THE PICKER DEFAULT, MEMOISED. `{tenant: (stamp, table_key)}`, the shape -#: `scope_cache` stores. -#: -#: ⛔ WHY THE DERIVED STRING AND NOT THE DOCUMENT. The obvious cache here is the `user_tables` -#: bucket itself, and it is the wrong one: that bucket is up to 35.8 MB per tenant and this box is -#: the HF free tier, so caching it would trade a latency problem for a memory one. What the warm -#: path actually needs is `discover_default_table`'s ANSWER — one short string. -#: -#: ⛔ AND WHY NOT A PLAIN `_BOARD_RETIRED`-STYLE ONCE-PER-PROCESS SET, which would have been less -#: code: the election reads which profile databases exist and which hold rows, and BOTH change -#: while the process lives (somebody creates a database, an automation writes the first row). A -#: once-per-process memo would pin the picker's default to whatever was true at boot and never -#: correct itself — a default that disagrees with the save door, which is exactly the wave-25 R2 -#: defect the `"table"` line's own comment records. A TTL bounds the staleness instead. -#: -#: ⚠ `scope_cache` rather than a hand-rolled dict, because it is the house pattern for precisely -#: this (`routes_customers`, `routes_products`, `pages` all use it) and it is stale-while-refresh: -#: once a copy exists NO request blocks on a rebuild. Automation was the one module importing it -#: nowhere, which the wave-30 scout named as the reason every other surface feels fast. -_DISCOVER_DEFAULT = {} -#: 5 minutes — the same order as `apiBridge.ts:CUSTOMERS_FRESH_MS` on the client. Overridable so a -#: gate can pin it rather than sleep. -_DISCOVER_DEFAULT_TTL = float(os.environ.get("AIOS_AUTOMATION_DEFAULT_TTL") or 300) - - -#: The one store key `_LentTables` intercepts. DERIVED from the engine's own constant rather than -#: written out here: `engine.ut_all` reads the bucket through it, so if that key ever moves, the -#: lend moves with it instead of silently becoming a pass-through that still looks correct. -_UT_STORE_KEY = engine.UT_STORE_KEY - - -def _LentTables(runtime, tables): - """⭐⭐ WAVE 30 · T13, NOW W31-C1 — a read-only `st` that serves ONE already-read `user_tables` - document and passes every other key straight through to the real runtime. - - ⭐⭐ WAVE 31 · C1 — THE BODY IS NOW `core.user_tables.lend`, AND THE CLASS THAT USED TO BE HERE - IS GONE. W30's version carried its own note that the general fix was unavailable at the time: - *"adding a `tables=` parameter to `may_open` would mean editing platform/core/user_tables.py, - which belongs to another lane this wave."* It is session B's lane THIS wave, they built the - generalisation (`user_tables.lend` + the `_LENDABLE` allow-list) for W31-T10, and this is the - second caller adopting it rather than becoming a third copy of the shape. - - ⭐ AND THE SWAP FIXES SOMETHING THIS FUNCTION NEVER COVERED. The old class intercepted exactly - one key, so `may_open`'s LAST branch — `shares.may_see` → `role_for` → `st.get('object_shares')` - — still took a fresh read PER TABLE. `_LENDABLE` covers both buckets, so a shared database no - longer costs a second whole-document read per row of the picker. The name is kept because - `verify_automation`'s NC66 pins this symbol as the thing it replaces to restore the `1 + N` - state; keeping it a function means that control still has exactly one seam to swap. - - ⛔ THE OBVIOUS FIX IS STILL THE FORBIDDEN ONE. Inlining the creator-or-admin test here would - remove the N reads and re-create the exact defect wave 20 fixed: this route USED to carry its - own wider rule (`createdBy in (uname, 'automation', 'scheduler')`), so a non-admin saw a - database in the picker and was refused the moment they opened it. `may_open` stays THE one - resolver, unmodified and still called per table; it is simply no longer charged for a document - the caller is already holding. - """ - # ⚠ IMPORTED HERE, not at module scope — `core` is deliberately kept out of this module's - # import-time graph (the same reason `automation_tables` does it locally 300 lines down). - import core.user_tables as _ut - return _ut.lend(runtime, **{_UT_STORE_KEY: tables}) - - -def _discover_default(session, tables): - """The discovery picker's default table for this tenant — WITHOUT a bucket read on a warm call. - - ⚠ `tables` is the document the caller ALREADY holds on a cold call (the retirement pass reads - one). Passing it through means the cold path elects from the copy it has rather than taking a - second one, so this is never an extra read — only ever a saved one. - """ - if not session.runtime.available(): - return "" - if tables is not None: - # Cold call: the document is in hand. Elect from it and prime the memo in the same pass. - value = engine.discover_default_table(session.runtime, tables=tables) - _DISCOVER_DEFAULT[session.tenant] = (time.time(), value) - return value - return scope_cache.get(_DISCOVER_DEFAULT, session.tenant, _DISCOVER_DEFAULT_TTL, - lambda: engine.discover_default_table(session.runtime)) - - -def warm_default(rt): - """⭐⭐ WAVE 30 · T12, THE COLD HALF. Elect the discovery picker's default at BOOT. - - ⛔ THE HALF THE MEMO CANNOT FIX, and it is why this exists rather than being a nicety. The memo - above makes calls 2..N free; **call 1 is still a full `user_tables` download**, and it lands on - whoever clicks Automation first after a deploy — the one visitor with no cache anywhere, - waiting on a document whose documented ceiling is 35.8 MB / ~1.4 s, taken under the store's - single lock. That is the owner's *"only automation has a loading screen"* on a cold Space, and - two waves of memoising the warm path could never touch it. Automation was the only module - importing `scope_cache` nowhere AND the only one absent from `_prewarm`. - - ⭐ IT ELECTS, IT DOES NOT CACHE THE DOCUMENT — deliberately, and this is the whole design. - `discover_default_table` reads the bucket once here, in the prewarm daemon thread where nobody - is waiting, and what survives is a DERIVED STRING. Holding the 35.8 MB document resident would - trade a latency the tenant notices for memory the HF free tier does not have, which is the - argument `_DISCOVER_DEFAULT`'s own note makes against caching it. - - ⚠ SAME TTL AS THE REQUEST PATH, not a permanent set: the election reads which databases exist - and which hold rows, and both change while the process lives. Priming by assignment is exactly - what the cold request path already does one function up, so there is one way this memo is - filled, not two. - - Returns the elected key (`""` when the store is unavailable), so a caller can log it rather - than guess whether the warm-up did anything. Called from `main.py:_prewarm` only. - """ - tenant = str(getattr(rt, "key", "") or "") - if not tenant or not rt.available(): - return "" - value = engine.discover_default_table(rt) - _DISCOVER_DEFAULT[tenant] = (time.time(), value) - return value - - -@router.get("/automations") -def list_automations(session: Session = Depends(_GATE)): - """`{automations: [...], kinds: [...], cronPresets: [...]}` — the rail's whole payload. - - The vocabularies ride WITH the list rather than sitting in a client constant: the cron - presets and the kind list are the server's, and a client copy of either is a thing that goes - stale silently (the editor would offer a preset the parser rejects).""" - # ⭐⭐ WAVE 29 (W29-T01, owner item 5: "Automation takes a while to appear"). THE COMPLAINT WAS - # THIS ROUTE, and the cost was never the automations: it read the whole `user_tables` bucket - # THREE TIMES per call — once for the stage-field retirement scan, twice more inside - # `discover_default_table`. That bucket's documented ceiling is 35.8 MB / ~1.4 s to serialize - # (`automation_engine.py` header) and `core/store.py` re-serializes on EVERY `.get()`, hit or - # miss, UNDER THE STORE LOCK — so the reads also serialize behind each other. ~4 s of deep - # copying to render a rail that shows a name and a toggle. - # - # Now: ONE read, lent to both callers. And the retirement SCAN — O(all row-cells in the - # tenant), which hits its own `if not stage_keys: return` only AFTER walking every row of - # every table — runs once per tenant per process, mirroring the `_C8_MIGRATED` guard below. - # - # ⛔ WHY SKIPPING THE SCAN ON CALLS 2..N IS SAFE, and it is not "because it is idempotent": - # `all_definitions` STRIPS retired board state on every read (`automation_engine.py`'s - # `_without_retired_board`), so the persist step is hygiene, not correctness. No client can - # ever be shown state this skip left behind. Nothing writes new `stage_auto_` cells either — - # wave 26's R6 deleted the board that made them. - # ⭐⭐ WAVE 30 · T12 (owner items 4/5, the complaint that has now survived TWO waves). - # ⛔ W29-T01 GUARDED THE SCAN AND NOT THE READ, and that is the whole of what was left. The - # line below used to be unconditional — every single request deep-copied the tenant's entire - # `user_tables` document (documented ceiling 35.8 MB / ~1.4 s, and `core/store.py:Store.get` - # re-serializes on EVERY `.get()`, hit or miss, UNDER THE STORE LOCK so the copies also queue - # behind each other) — while on calls 2..N the result was DISCARDED: `tables` had exactly two - # consumers, the `_BOARD_RETIRED` scan (skipped after call 1) and a picker DEFAULT STRING. - # A whole-tenant document, per request, to render a rail showing a name and a toggle. - tables = None - if session.runtime.available() and session.tenant not in _BOARD_RETIRED: - tables = engine.ut_all(session.runtime) - _BOARD_RETIRED.add(session.tenant) - # Idempotent Board retirement removes only engine-marked stage fields and legacy Board - # state. User-created Status/Stage columns remain intact. - engine.retire_automation_board_state(session.runtime, tables=tables) - # ⭐⭐ WAVE 35 · T37 / OWNER RULING R10 — ROYAL IMPORTS' STATEMENTS AGENT EXISTS BEFORE ANYBODY - # ASKS FOR IT. It is minted ONCE, by the container, SWITCHED OFF; every subsequent call finds - # it present and returns immediately. See `engine.seed_statements_agent` for why this one is - # STORED where wave 34's Odoo agent is derived, and why the row's existence is a sufficient - # idempotency key (it cannot be deleted). - # ⚠ GUARDED SO IT CAN NEVER TAKE THE RAIL DOWN. A tenant with no Odoo, a store mid-outage or a - # validator change must degrade to "no statements agent", never to a 500 on the one route the - # whole Agents surface polls. The same posture `_odoo_sync_row` takes for the same reason. - if session.tenant not in _STATEMENTS_SEEDED: - try: - engine.seed_statements_agent(session.runtime, username=session.uname or "system") - except Exception: # noqa: BLE001 - pass - _STATEMENTS_SEEDED.add(session.tenant) - defs = engine.all_definitions(session.runtime) - items = [_wire(d, session.tenant, rt=session.runtime) for _, d in - sorted(defs.items(), key=lambda kv: (kv[1].get("name") or "").lower())] - # ⭐⭐ WAVE 34 · CONTRACT C3 (W34-T48) — field agents join the list as SYNTHETIC rows. - # ⚠ MERGED AND RE-SORTED, not appended in a block at the end. R13 asks for a field agent to be - # "VIEWABLE under the Agent module", i.e. one list of agents, not a list with a second list - # stapled to it — and a rail that sorts by name everywhere except its last few rows reads as a - # rendering bug. `_field_agent_rows` costs no whole-document read; see its docstring. - _sys_rows = [r for r in (_odoo_sync_row(session),) if r] - items = sorted(items + _field_agent_rows(session) + _sys_rows, - key=lambda r: str(r.get("name") or "").lower()) - return {"automations": items, - # ⭐ WAVE 24 — DERIVED from the engine's own `KINDS`, not a hand-written trio. It was - # three literals that happened to match, which is a second copy of a server - # vocabulary; R6 has just made two of them uncreatable and `plain` has joined, so a - # hand list would now be wrong in three ways at once. `creatable` carries R6 onto the - # wire, so the ruling is a fact the client can read rather than one it must remember. - "kinds": [{"key": k, "label": engine.KIND_LABELS.get(k, k), - "creatable": k not in engine.RETIRED_KINDS} - for k in engine.KINDS], - "cronPresets": engine.CRON_PRESETS, - # ⚠ A BOOLEAN, NEVER THE KEY. The surface needs to say "the paid rung is not - # configured" honestly instead of offering a tier that will silently refuse — and - # that needs exactly one bit. Shipping the key itself to a browser would put a - # billable secret in every user's devtools. - "paidReady": engine.bd_ready(), - # THE SOURCE REGISTRY (D-9's seam), on the wire for the same reason `cronPresets` is: - # the module that RUNS a source is the only thing entitled to say what it can do, and - # a client copy of "Instagram can discover, TikTok cannot" goes stale in silence. - "sources": engine.source_status(), - # The discovery vocabulary, likewise server-owned: every name here is MEASURED- - # accepted by the vendor's own validator, so a client that invented one would build a - # query the API rejects. `lead` is the subset seen carrying VALUES on real rows. - # ⭐⭐ WAVE 32 · T46 (D-167) — THE VOCABULARY HAS A PLATFORM, AND `byKind` IS ADDITIVE - # ON PURPOSE. `fields`/`lead` keep INSTAGRAM's 21 and 3, so no stored automation and no - # client that has not adopted this changes behaviour today; `byKind` carries the per- - # corpus answer, and the SERVER already refuses a TikTok predicate naming one of the 16 - # fields TikTok's dataset does not have (`clean_predicates(..., kind)`). The door is - # closed either way — this is what lets the Find panel stop OFFERING them. - # ⚠ Derived through `engine.filter_fields`, the same accessor the validator uses, so - # the published vocabulary and the enforced one cannot drift — which is precisely what - # D-167 was: a route serving 21 names and a validator checking the same 21, both wrong - # about TikTok together, with nothing able to notice. - "discoverByKind": {k: {"fields": list(engine.filter_fields(k)[0]), - "lead": list(engine.filter_fields(k)[1])} - for k in engine.DISCOVERY_KINDS}, - "discover": {"fields": list(engine.BD_FILTER_FIELDS), - "lead": list(engine.BD_FILTER_LEAD), - "operators": list(engine.BD_FILTER_OPS), - "nullaryOperators": list(engine.BD_NULLARY_OPS), - "maxRecords": engine.BD_MAX_RECORDS, - # Wave 22 C4 — ADDITIVE: per-field flags for the toggle rows (the 3 - # PII fields stay structurally absent, R3) + the too-big guard's - # numbers, so the surface can say WHY a filter is refused before the - # server has to. - "filterMeta": engine.filter_meta(), - "guard": {"minNarrowing": engine.BD_MIN_NARROWING, - "maxRecords": engine.BD_MAX_RECORDS}, - # ⛔ D-59's `categoryOptions` IS DELIBERATELY NOT HERE — it is served by - # `GET /automations/discover/categories`. It sat on this payload for one - # commit and that was a real defect: THIS ROUTE IS POLLED EVERY 2.5 s by - # the whole automation surface, and deriving the options reads two user - # tables plus the PLATFORM MASTER, which is a different HF repo — i.e. a - # network round-trip per poll, per open tab. Caught by the gate's own - # output, which started carrying `store:get:master_snapshots` errors from - # a suite whose stated contract is that it touches no network. - # C6/R5's vocabulary, so the seed picker cannot offer a source the - # validator refuses. - "seedSources": list(engine.SEED_SOURCES), - "seedMaxRows": engine.SEED_MAX_ROWS, - # W29-T01: `tables` is the snapshot read once at the top of this handler. - # ⚠ IT WAS TAKEN BEFORE THE RETIREMENT WROTE, and that is deliberate and - # harmless: retirement only removes `stage_auto_*` FIELDS and pops those - # same keys off rows, and the election reads the preset-profile vocabulary - # (`handle` + N preset keys) and whether a table has ANY rows. A stage key - # is in neither set, and no row is ever deleted — so the pre-write - # snapshot and the post-write bucket cannot elect different tables. - # 2026-08-10 — the OFFER must name the table the SAVE will actually use. - # This was the bare `DISCOVER_TABLE` constant while `create`/`patch` now - # resolve a targetless discovery flow to the profile database the tenant - # already has, so the picker would have shown `ut_ig_candidates` and the - # save would have written somewhere else — a default that disagrees with - # itself across two panels, which is the shape wave 25's R2 fixed for - # `targetTable` vs the action's `table`. - # ⭐ WAVE 30 · T12 — through the per-tenant memo. The ELECTION rule and - # everything the comment above says about it are unchanged; what changed - # is that a warm request no longer re-reads a 35.8 MB document to - # recompute a string that did not move. - "table": (_discover_default(session, tables) - or engine.DISCOVER_TABLE)}, - "storeAvailable": bool(session.runtime.available()), - # Wave 22 C3 — the trigger vocabulary, session-scoped because email readiness is a - # per-USER fact (the poll runs through the creator's own Gmail connection). - "triggers": _triggers_vocab(session), - # ⭐ WAVE 23 C4 (R3) — the ACTION MENU, including what we have not built. Each row - # carries `ready`, so B paints "Send email" and "Run script" faded with the server's - # own reason instead of omitting them — the owner asked for Airtable's full menu, and - # a shorter list would imply those actions do not exist. `clean_actions` REFUSES an - # unready kind, so the faded state is a wall rather than a styling choice. - # ⭐ W35-T35 (C8): `session.runtime` is what withholds a TENANT-GATED row. Passing it - # here is the whole of "the menu does not offer Send statements to another tenant"; - # the STORE-side wall is `clean_actions(rt=)`, deliberately separate, because a picker - # is not a security boundary [[opening-a-route-widens-every-field]]. - "actionsCatalog": engine.action_catalog(session.runtime), - # The builder's own vocabulary: how deep a condition tree may nest, how deep groups - # may nest, and the ceilings. B reads these instead of hard-coding the same numbers - # into its "+ Add condition" affordance. - "flow": {"condOps": list(engine.LANE_OPS), - "nullaryCondOps": list(engine.LANE_NULLARY_OPS), - "maxCondDepth": engine.MAX_COND_DEPTH, - "maxCondChildren": engine.MAX_COND_CHILDREN, - "maxGroupDepth": engine.MAX_GROUP_DEPTH, - "maxActions": engine.MAX_ACTIONS, - # ⭐ W29-T09 (owner item 11) — THE POST-GROUP VOCABULARY, on the wire for the - # same reason `cronPresets` is: `clean_post_groups` REFUSES a type it does - # not know, so a client that invented one would build a config the save door - # rejects with a sentence about a word the person never typed. Both halves - # ride — the stored key and the label a person reads — because a client-side - # translation of `video` into "Reels" is a second copy of this list. - "postTypes": [{"key": t, "label": engine.POST_TYPE_LABELS.get(t, t)} - for t in engine.POST_TYPES], - # The ceiling a group's limit is judged against. `clean_post_groups` bounds a - # group by the action's OWN `maxPosts`, not by a constant, so the control can - # only warn honestly if it reads the same number the validator uses. - "maxPostsPerPull": engine.MAX_POSTS_PER_PULL, - # ⭐⭐ W33-T52 (owner item 16) — WHICH CONFIG KEYS A KIND REQUIRES, on the - # wire, for exactly the reason `postTypes` above is: the panel paints a red - # `*` beside a required control, and a client-side table of which keys those - # are would be a SECOND copy of `ACTION_REQUIRED` living in another file. The - # two would agree on the day they were written and diverge the first time a - # kind gained a key — the panel would then mark a control optional that the - # runner blocks on, and the person would read "this is fine" from the one - # surface that is meant to tell them it is not. - # ⛔ BOTH HALVES RIDE, phrase AND key, because the phrase is the ONLY human - # wording of that requirement anywhere ("a column to write into"), and the - # runner's own refusal sentence is built from it. A client that re-worded it - # would give the same requirement two names. - # ⚠ It is `ACTION_REQUIRED`, not `WEB_REQUIRED`: the table is the one the - # run refusal and `unconfigured_actions` already read, so a kind added to it - # later paints its `*` with no client change at all. - "actionRequired": {kind: [{"phrase": phrase, "key": key} - for phrase, key in reqs] - for kind, reqs in engine.ACTION_REQUIRED.items()}}, - # Wave 21 C6-A1: {"enabled": bool, "source": "in-process"|"external"|""} — see - # `_tick_state` for why one boolean off AIOS_AUTOMATIONS alone would lie. - "tick": _tick_state()} - - -@router.post("/automations/discover/estimate") -def discover_estimate(body: dict = Body(default=None), session: Session = Depends(_GATE)): - """What would this search cost? Shown BEFORE the run, never after. - - ⚠ THE ANSWER IS AN ESTIMATE AND SAYS SO IN ITS OWN PAYLOAD (`basis: "SPEC"`). Bright Data - never returns a price before a run — the funds gate fires first and `price: 0` means "not - priced", not "free" — and this account's token cannot read a balance (`/customer/balance` - answers 403). A number presented as billed would be the invented measurement this whole - module refuses to make. - """ - return engine.discover_estimate((body or {}).get("recordsLimit")) - - -#: C8's migration ran, per tenant, this process. Once is the point: the sweep only writes when -#: an unbound bag exists, so after the first pass this is a read that finds nothing. -_C8_MIGRATED = set() - - -def _table_views(session, key): - """⭐ WAVE 24 (item 8) — the saved views on ONE user table as `[{id, label}]`, or **None** - when the workspace bucket could not be read at all. - - The client said "This server did not offer a view list" because the payload genuinely had - no `views` key. C-TYPES: **absent stays ABSENT.** `[]` is a MEASUREMENT ("this database has - no saved views") and `None` is a STATE ("nobody looked") — collapsing them is precisely how - an empty picker comes to read as a claim about the database. - - ⛔ SOURCED FROM `core.table_store`, WHICH IS THE READER `view_filter` ALREADY USES — not a - second one, the same one, listed instead of looked up. The brief said to source it the way - the grid does; the grid's projection (`aios_grid.views_from_defs`) is the wrong list HERE and - the difference matters: it INJECTS the system view ("All records") and PROJECTS cohorts as - views, and neither of those lives in the stored bucket — so `view_filter` answers "view no - longer exists" for every one of them. A picker built from that projection would offer options - the `enters_view` trigger cannot resolve, which is a worse bug than the missing key it fixes. - (An `enters_view` trigger on "All records" would also mean "fire on every record", so its - absence is correct rather than a gap.) - - ⚠ `consume_corrections=False`: this is a READ for a picker, and the default MUTATES — it - takes and clears the pending field-correction acknowledgement, so listing views would eat a - protocol message meant for the grid. - """ - try: - import core.table_store as table_store - # ⛔ THE REACHABILITY PROBE IS NOT REDUNDANT, and leaving it out was a real defect this - # gate's own NC caught: EVERY `TableStore` accessor wraps its store read in `try/except` - # and returns `{}` on failure. So a bucket that could not be read is indistinguishable - # from one with no views — which collapses exactly the ABSENT/EMPTY distinction this - # function exists to preserve, and the caller would ship `views: []` as a measurement - # nobody took. The one read that is allowed to raise has to be ours. - session.runtime.get(f"{key}_table_workspace") - tops = table_store.make(f"{key}_table_workspace", st=session.runtime) - own = (tops.workspace(session.uname, consume_corrections=False) or {}).get("views") or {} - shared = tops.shared_views(session.uname, session.admin) or {} - except Exception: # noqa: BLE001 - return None - merged = {**shared, **own} # a view lives in ONE home; the merge is belt-and-braces - return sorted( - ({"id": str(vid), "label": str((v or {}).get("name") or vid)} - for vid, v in merged.items() if isinstance(v, dict)), - key=lambda r: r["label"].lower()) - - -@router.get("/automations/tables") -def automation_tables(session: Session = Depends(_GATE)): - """The blank databases an automation can target, with their fields. - - ⚠ A READ-ONLY MIRROR of C3-UT's `GET /api/v1/tables`, under this router's own prefix so the - two can never collide. It exists because the automation editor needs the table+field list to - build a config at all, and D must not block on A's route landing. When C3-UT is live this - keeps working (same bucket) — it is a duplicate reader, never a second writer. - - ⛔ WAVE 20, item 3 — IT NO LONGER MIRRORS THE WALL, IT CALLS IT. This route re-implemented - `may_open` and got it WIDER: it also admitted `createdBy in ('automation', 'scheduler')`, so - a non-admin saw automation-created databases here and was refused the moment they opened, - edited or deleted one. A duplicate READER is fine; a duplicate WALL is not, because the two - only disagree in front of a user. One resolver now, and the engine stamps a human owner so - the merge takes nothing legitimate away (`ut_ensure`, `MACHINE_OWNERS`). - """ - import core.user_tables as user_tables - - # C8 (wave 22): bind pre-law automation bags to the definitions that write them — once - # per tenant per process, and a no-op read after the first real pass. A write-on-read, - # stated out loud: this is the surface whose stale bags mislead (the column picker), so - # it is where the truth gets repaired. - if session.tenant not in _C8_MIGRATED and session.runtime.available(): - _C8_MIGRATED.add(session.tenant) - try: - engine.bind_unbound_fields(session.runtime) - except Exception: # noqa: BLE001 - pass - - out = [] - # ⭐⭐ WAVE 30 · T13 — ONE read of the tenant document, LENT to the wall for every table. - # It was `1 + N` full deep copies (this `ut_all`, then `may_open` → `get` → `all_tables` per - # table), each of a document with a 35.8 MB / ~370 ms ceiling, all of them queued behind - # `Store._lock`. `may_open` is unchanged and still asked about every table — see `_LentTables` - # on why re-implementing the wall here is the one fix that is NOT available. - _tables_doc = engine.ut_all(session.runtime) - _lent = _LentTables(session.runtime, _tables_doc) - for key, t in sorted(_tables_doc.items(), - key=lambda kv: (kv[1].get("label") or "").lower()): - if not user_tables.may_open(key, session.uname, session.admin, st=_lent): - continue - row = {"key": key, "label": t.get("label") or key, - "source": t.get("source") or "Blank", - "rowCount": len(t.get("rows") or {}), - "fields": [{"key": f.get("key"), "label": f.get("label"), - "type": f.get("type") or "text", - "automation": f.get("automation") or None} - for f in (t.get("fields") or [])]} - views = _table_views(session, key) - if views is not None: - row["views"] = views # absent when the bucket did not answer — never [] - out.append(row) - return {"tables": out} - - -@router.get("/automations/presets") -def automation_presets(table: str = "", session: Session = Depends(_GATE)): - """⭐ WAVE 25 · C1 / R2b — the Instagram preset columns, diffed against ONE database. - - `{fields: [{key, label, type, present}], willUse: [...], willCreate: [...]}` — the - "already in this database" / "will be created" split the owner asked for, so the Create record - action's configuration can SHOW what pointing it here does before anything is spent. - - ⛔ DECLARED ABOVE `/automations/{auto_id}`, AND THAT IS LOAD-BEARING RATHER THAN TIDY. - FastAPI matches routes in DECLARATION order, so a literal path registered after a sibling - path-parameter route is never reached — this handler would simply never run and - `get_automation` would answer "no automation with that id" for the id `presets`. A 404 with a - plausible sentence is the worst possible failure here, because it reads as "the endpoint is - fine, the data is missing". `/automations/tables` sits above the same route for the same - reason; this follows it rather than inventing a second convention. - - ⚠ WALLED BY `may_open`, LIKE EVERY OTHER TABLE READER. Without it, asking about a table you - cannot open would answer which of its columns exist — a small disclosure, and exactly the - duplicate-wall mistake `automation_tables` records at wave 20. - """ - import core.user_tables as user_tables - - key = str(table or "").strip() - if key and not user_tables.may_open(key, session.uname, session.admin, - st=session.runtime): - raise err(404, "unknown_table", "no such database") - return engine.preset_plan(session.runtime, key) - - -@router.get("/automations/discover/categories") -def automation_categories(session: Session = Depends(_GATE)): - """⭐ DEBT D-59 — the Category combobox's options: the values this deployment has ACTUALLY - SEEN, each with its observed count. `{options: [{value, count}]}`. - - ⛔ ITS OWN ROUTE, ON PURPOSE. This belongs to the Find surface and is asked for when that - panel opens — it must never ride `GET /automations`, which the whole automation surface polls - every 2.5 s: deriving these reads two user tables AND the platform master (a different HF - repo), so on the polled payload it is a network round-trip per tab per poll. - - ⚠ THE CONTROL STAYS A COMBOBOX. These are the values we have seen, not the values that exist. - D-59 is explicit that transcribing Instagram's published taxonomy would be worse than no - dropdown — a filter on a value the corpus does not use returns zero rows and looks exactly - like an honest "no such accounts exist", which misleads precisely when it looks authoritative. - """ - return {"options": engine.observed_categories(session.runtime)} - - -@router.post("/automations/seed/derive") -def automation_seed_derive(body: dict = Body(default=None), session: Session = Depends(_GATE)): - """⭐ WAVE 25 · C6 / R5 — "find me more accounts like the ones in this view". - - Body `{source: "view"|"cohort", table: "", id: ""}` → - `{derived: [...], basis: {rows, fields:[{name,label,value,coverage}], related, note}}` - - ⛔ IT DERIVES AND RETURNS; IT SAVES NOTHING. R5: the derived conditions are "visible and - editable, never hidden" — so the client drops them into the ordinary condition rows, where the - user edits them like anything else, and the ordinary PATCH stores them. A door that both - derived and saved would make the suggestion feel like a decision. - - ⚠ `basis` IS NOT DECORATION AND MUST BE RENDERED. It carries how many rows were read and the - MEASURED coverage of each characteristic, which is the difference between "12 of 12 of these - bios say florist" and "7 of 12 do" — presented identically, the weak one reads as authority. - - ⚠ Declared ABOVE `/automations/{auto_id}` for the reason `/automations/presets` is (FastAPI - matches in declaration order); that ordering is gated. - """ - import core.user_tables as user_tables - - body = body or {} - table = str(body.get("table") or "").strip() - if not table: - raise err(400, "no_table", "name the database to read the seed records from") - if not user_tables.may_open(table, session.uname, session.admin, st=session.runtime): - raise err(404, "unknown_table", "no such database") - source = str(body.get("source") or "view").strip().lower() - if source not in engine.SEED_SOURCES: - raise err(400, "bad_seed_source", - f"a seed comes from one of: {', '.join(engine.SEED_SOURCES)}") - rows, problem = engine.seed_rows(session.runtime, table, str(body.get("id") or "")) - if problem: - raise err(400, "bad_seed", problem) - derived, basis = engine.seed_predicates(rows) - return {"source": source, "table": table, "id": str(body.get("id") or ""), - "derived": derived, "basis": basis} - - -#: The injected chat transport, for gates only — `None` in every shipped path, so the route takes -#: the real ladder. Set by `verify_automation.py` to prove this door end to end with NO API key and -#: NO spend (`routes_query._call_model`'s `chat` argument is the same idea and the same reason). -_DRAFT_CHAT = [None] - - -@router.post("/automations/draft") -def draft_automation(body: dict = Body(default=None), session: Session = Depends(_GATE)): - """⭐⭐ WAVE 33 · W33-T54/T55 (owner item 7, ruling R3) — A DESCRIPTION BECOMES A DRAFT FLOW. - - `{prompt}` → `{draft: {name, trigger, table, actions:[{id,kind,config,why}]}, provider, - dropped: [...], saved: false}` — or a 400 carrying ONE plain sentence. - - ⛔⛔ NOTHING IS SAVED HERE, AND THE `saved: false` ON THE WIRE IS NOT DECORATION. T54's - `done-when` is *"gets back a DRAFT flow they can see BEFORE anything is saved"*; a door that - wrote first and showed second would satisfy every check about the flow's contents and none about - the promise. Accepting a draft is the ORDINARY `POST /automations` — R3's *"indistinguishable - from a hand-built one"* is achieved by there being no second write path, not by making the - second one look similar. - - ⛔⛔ CONTRACT C7, AND THIS ROUTE IS WHERE IT IS ENFORCED: *"the output must pass `clean_actions` - unchanged."* So it is RUN, here, before the person ever sees the draft — and the result is - DIFFED against what the model wrote. `clean_actions` has no disclosure channel (D-75): it drops - a config key it does not recognise and answers 200. Without this diff a person would accept a - flow, open it, and find a step configured differently from the one they were shown, with nothing - anywhere having said so. `dropped` is that channel, built here because this is the first caller - that needed one. - ⚠ A draft whose actions `clean_actions` REFUSES outright is a 400, never a partial flow: half a - flow presented as a whole one is the one outcome worse than a refusal. - - ⚠ DECLARED ABOVE `/automations/{auto_id}`, and that is load-bearing rather than tidy — FastAPI - matches in DECLARATION order, so a literal path registered after a sibling path-parameter route - is never reached, and `get_automation` would answer *"no automation with that id"* for the id - `draft`. The same reason `/automations/tables` and `/automations/presets` sit up here. - """ - prompt = str((body or {}).get("prompt") or "").strip() - if not prompt: - raise err(400, "no_prompt", "type what you want the automation to do") - # ⭐⭐ ASK D-18 (2026-08-18) — THE KEY THE CLIENT SENDS IS NOW READ. This door took `prompt` off - # the body and nothing else, so the Agent chat's model toggle (W36-T34) was a control that - # configured nothing: the value was accepted and dropped, which is the shape three tickets in - # this wave already tripped over. D held T34 rather than mark a picker BUILT while its value - # stopped at the browser, and was right to. - # ⚠ UNKNOWN OR UNCONFIGURED FALLS BACK TO THE LADDER, never refuses (D-18's asked-for posture, - # and C5's everywhere else): a model the ladder stops offering must not turn every later draft - # into an error. The response already names the rung that ANSWERED, which is the honest half. - _model = str((body or {}).get("model") or "").strip().lower() - if _model in ("", "auto"): - _model = None - # ⚠ THE TENANT'S OWN TABLES, THROUGH THE EXISTING WALL. `automation_tables` applies `may_open` - # per table, so the model is shown exactly the databases this caller may already see and cannot - # name one they were not granted — the permission wall re-used, never a second one built beside - # it. (`QueryPage`'s `granted` prop carries the same clause on the client side.) - tables = (automation_tables(session) or {}).get("tables") or [] - # ⭐⭐ WAVE 34 · W34-T42 / R18 — THE DRAFTER IS OFFERED WHAT THE PICKER OFFERS, nothing more. - # - # R18 collapsed six web actions into one on the menu. The other five stay in the catalog - # because the surviving "Web agent" composes its own steps out of them server-side — but a - # DRAFT is read and accepted by a PERSON, who then edits it on the Canvas. Handing the model - # the unfiltered catalog would let it draft a step kind that is not in the action picker: the - # reader could not add a second one, could not reason about where it came from, and the flow - # they accepted would not be one they could have built by hand. R3's *"indistinguishable from - # a hand-built one"* is a claim about what a person can REACH, not only about the write path. - # ⚠ This is the ONE place the filter is applied on this route, and it is applied to the - # drafter's vocabulary only. `clean_actions` below still validates against the FULL catalog, - # so a flow that already holds a hidden kind is unaffected and nothing 400s (D-65). - # ⚠ AND `required` IS NARROWED WITH IT, which is the half that is easy to miss. `draft_flow` - # builds the ENUM from `catalog` and the "settings you must write" prose from `required`, so - # filtering one and not the other describes settings for kinds the model cannot choose. That is - # not merely untidy: it spends prompt on unreachable options and invites the model to reach for - # one. `_ai_agent_plan` already narrows both in exactly this way; this follows it. - # W35-T35 (C8): tenant-gated rows are withheld here too, or the DRAFTER would offer a kind - # the same tenant's own save door then refuses — a flow the assistant proposes and the product - # will not accept. - menu_catalog = [r for r in engine.action_catalog(session.runtime) if r.get("menu") is not False] - _offered = {r["kind"] for r in menu_catalog} - draft, refusal, provider = ai_review.draft_flow( - prompt=prompt, - catalog=menu_catalog, - required={k: v for k, v in engine.ACTION_REQUIRED.items() if k in _offered}, - triggers=_triggers_vocab(session), - tables=tables, - # ⭐ W35 · CONTRACT C7 (`NOTE E-16`) — the spend is ATTRIBUTED. Unlike the two engine call - # sites, this one has a real person behind it: somebody typed the sentence, so `user` is - # the caller rather than the automation's owner. - st=session.runtime, user=getattr(session, "uname", "") or "", - model=_model, - chat=_DRAFT_CHAT[0]) - if refusal or not draft: - raise err(400, "draft_refused", refusal or "no automation could be drafted from that") - - # ── C7: run the real save-door validator and DIFF it ────────────────────────────────── - wrote = draft.get("actions") or [] - # W35-T35 (C8): the SAME `rt` the menu was built with. Without it this validator would refuse - # a tenant-gated kind it had just offered the model — the draft door disagreeing with the save - # door about one tenant, which reads as "the assistant produced an invalid flow". - cleaned, why = engine.clean_actions([{"kind": a.get("kind"), "config": a.get("config") or {}} - for a in wrote], rt=session.runtime) - if why or cleaned is None: - raise err(400, "draft_invalid", - f"the assistant produced a flow this deployment will not accept: {why}") - dropped = [] - for before, after in zip(wrote, cleaned): - b_cfg = before.get("config") or {} - a_cfg = after.get("config") or {} - gone = sorted(k for k in b_cfg - if k not in a_cfg or str(a_cfg.get(k)) != str(b_cfg.get(k))) - if gone: - dropped.append({"kind": str(after.get("kind") or ""), "keys": gone}) - # ⛔⛔ WHAT IS STILL MISSING FROM EACH STEP, PER STEP — and this is the fix for the sharpest - # thing the verifier found. The system prompt TELLS the model *"leave one blank rather than - # inventing a web address, a CSS selector or a column name"* (which is right), and the panel - # skips empty values (which is also right, on its own). Together they meant a `web_read` with - # no selector and no target column painted as a FINISHED step — url, attr, timeout, nothing - # else — and the ordinary create door then accepted it. A person approved a step that reads - # nothing into nowhere, having been shown no blank at all. - # ⚠ `engine.action_needs` IS THE PREDICATE, not a second list: the same function the builder's - # Configured/Unconfigured label and the run refusal read. Three readers, one answer. - _tables_by_key = {str(t.get("key")): t for t in tables} - _target_cols = {str(f.get("key")) for f - in (_tables_by_key.get(str(draft.get("table") or "")) or {}).get("fields") or []} - shown = [] - for i, after in enumerate(cleaned): - row = dict(after, why=str((wrote[i] or {}).get("why") or ""), - needs=engine.action_needs(after)) - # ⛔ AND A COLUMN NAME THE DATABASE DOES NOT HAVE. `field` is a free string that nobody - # validated at draft, at accept or at store — so a model writing the column's LABEL - # ("Price") instead of its key ("price"), which is exactly what a person would say and an - # obedient model would echo, stored a write into a column that does not exist. No schema - # was violated; the flow was well-formed and did something other than what was asked. - _f = str((after.get("config") or {}).get("field") or "") - if _f and _target_cols and _f not in _target_cols: - row["unknownField"] = _f - shown.append(row) - # ⚠ A TRIGGER THE CREATE DOOR WOULD REFUSE IS CORRECTED HERE, NOT SHOWN AND THEN 400'd. The - # enum makes this need a model that ignores its own schema, but the failure mode is ugly: the - # draft paints, the person clicks Accept, and the save refuses with a sentence about a word - # they never chose. Falling back to `manual` and SAYING SO keeps the draft usable. - _trig = str(draft.get("trigger") or "").strip() - _notes = [] - if _trig and _trig not in tuple(engine.TRIGGER_KEYS): - _notes.append(f"the assistant asked for a trigger this deployment does not have " - f"({_trig}). Set to manual instead") - _trig = "" - _asked = int(draft.get("asked") or len(wrote)) - if _asked > len(shown): - _notes.append(f"the assistant wrote {_asked} steps and a draft carries at most " - f"{ai_review.MAX_DRAFT_ACTIONS}; the last {_asked - len(shown)} were not " - f"kept. Describe the job in two automations, or shorten it.") - # ⭐⭐ WAVE 34 · W34-T47 (mailbox D-5) — THE MODEL'S OWN WORDS ARE NORMALISED, BECAUSE R6 - # CANNOT BE ENFORCED BY A SOURCE SCAN. - # - # ⛔ `web_prose` READS FILES. Every string below is written by a language model at run time, - # so the gate is structurally blind to it: the sweep every lane did this wave is undone by our - # own drafter the first time it answers with an em dash. Lane D MEASURED that a prompt - # instruction does not hold (their system prompt says "Never use an em dash" and the very next - # cerebras turn came back with one), so this is code, at the boundary, not a nicer prompt. - # ⚠ THREE STRINGS REACH A SCREEN FROM HERE and all three are covered: the flow's NAME, each - # step's `why`, and the `dropped`/`notes` sentences. `W34-T42` is what made them visible. - # ⚠ A DIGIT RANGE IS A DIFFERENT SENTENCE: "10-20" means "10 to 20", and rewriting it as - # "10, 20" states two numbers where the model stated a span. It gets its own rule, first. - # Same shape as `routes_query._no_dashes`, deliberately: one behaviour, two doors. - # ⚠ MODULE-LEVEL, not a closure. A normaliser defined inside this handler is unreachable to - # anything that is not an HTTP request, so the only way to test it would be to drive the whole - # route and read the answer — which is how a rule ends up asserted by a grep instead of a call. - # ⛔ `dropped` IS NOT NORMALISED AND THAT IS DELIBERATE: it is `[{kind, keys}]`, an action kind - # from OUR catalog and config keys from OUR allowlist, never a model's sentence. The first - # version of this block mapped the normaliser over it and turned each dict into the string - # `"{'kind': ..., 'keys': [...]}"` — `verify_automation`'s own W33 section caught it with a - # `TypeError` on `d["kind"]`. Applying a text rule to a structure is how a channel stops - # carrying what its reader expects, and the reader here is a person's list of what changed. - shown = [dict(r, why=_draft_no_dashes(r.get("why"))) for r in shown] - _notes = [_draft_no_dashes(n) for n in _notes] - return {"draft": {"name": _draft_no_dashes(draft.get("name")) or "New automation", - "trigger": _trig or "manual", - "table": draft.get("table") or "", - "actions": shown}, - "provider": provider, "dropped": dropped, "notes": _notes, "saved": False} - - -# ══════════════════ WAVE 36 · W36-T38 (owner item 8, ruling R9) — AGENT-AUTHORED ACTIONS ══════ -# -# Owner, verbatim (2026-08-18): *"Create the ability for an automation agent to create any -# 'Action' under the Canvas, so its configuration can only be touched by the agent. Be it a tool a -# script etc. We need to really guardrail the reach of this script."* -# -# ⭐⭐ R9 IS A **WRITE** RULE, NOT A VISIBILITY RULE, and the ticket says so in as many words: the -# user MAY read an agent-authored action's configuration and MAY delete it; what they may not do -# is hand-edit it. Hiding a config from the person whose workspace it runs in is a different -# product, and a worse one — nobody can audit what they cannot see. -# -# ⛔ AND IT IS ENFORCED AT THE DOOR, NEVER IN A `disabled` ATTRIBUTE. `routes_automation` already -# carries that lesson for a different control; a client-side lock is a suggestion, and the whole -# point of R9 is that the agent's configuration stays coherent with what the agent believes it -# built. -# -# ⚠ WHY THE MARK IS A SEPARATE BUCKET RATHER THAN A KEY ON THE ACTION. `clean_actions` builds every -# action KEY BY KEY from an allowlist and drops anything it does not recognise (D-75) — so a -# marker stored inside the action would be silently erased on the next save, and the wall would -# quietly stop existing with every gate still green. This is an ANNOTATION layer keyed by -# `(automation id, action id)`; the configuration itself stays where it always was, in the -# automation, with exactly one writer. -AGENT_ACTIONS_KEY = "automation_agent_actions" - -#: Ids this door mints. ⚠ It must match `clean_actions`' `act_[a-z0-9_]{1,32}` or the engine -#: re-mints it and the annotation points at an action that no longer carries that id. -_AGENT_ACTION_PREFIX = "act_ag" - - -def _agent_marks(rt): - """`{auto_id: {action_id: mark}}` for one tenant. `{}` on any failure — an unreadable - annotation must degrade to "nothing is agent-owned", never to a 500 on the automations rail. - - ⛔ AND "DEGRADE TO NOTHING IS OWNED" IS THE SAFE DIRECTION HERE, which is worth stating because - it usually is not. The wall this feeds protects the AGENT's coherence, not the tenant's data: a - lost mark lets a person edit a config they own anyway, in their own workspace, on a step they - could always have deleted outright. A wall that failed CLOSED would instead make an automation - permanently unsavable because one annotation read timed out. - """ - try: - found = rt.get(AGENT_ACTIONS_KEY) or {} - except Exception: # noqa: BLE001 - return {} - return found if isinstance(found, dict) else {} - - -def _flat_actions(actions, out=None, depth=0): - """Every action in a flow, INCLUDING the ones nested inside a group's arms. - - ⚠ NESTED ACTIONS ARE THE ONES THIS MUST NOT MISS. `unconfigured_actions` walks them for the - same reason: a step inside an If/then branch is exactly the one a person cannot see, and a - wall that only looked at the top level would leave the agent's own nested step editable. - """ - out = [] if out is None else out - if depth > 6 or not isinstance(actions, list): - return out - for action in actions: - if not isinstance(action, dict): - continue - out.append(action) - for key in ("then", "else", "actions"): - _flat_actions(action.get(key), out, depth + 1) - return out - - -def _flow_actions(defn): - return _flat_actions(((defn or {}).get("flow") or {}).get("actions") or []) - - -def _same_action(left, right, rt): - """Do these two actions carry the SAME kind and configuration? - - ⛔ COMPARED AFTER `clean_actions`, ON BOTH SIDES, and that is the difference between a wall and - a nuisance. The stored action has already been through the cleaner; a client round-trip has - not, so it may carry a key order, a blank string or a dropped condition that means nothing. - Comparing raw shapes would refuse an edit that changes nothing, and a wall that fires on a - no-op is one an operator learns to route around [[one-question-two-normalizers]]. - """ - def _clean(action): - cleaned, error = engine.clean_actions([dict(action or {}, id="act_1")], rt=rt) - if error or not cleaned: - return None - one = dict(cleaned[0]) - one.pop("id", None) - return one - - a, b = _clean(left), _clean(right) - return a is not None and a == b - - -def _agent_owned_guard(session, auto_id, body): - """R9 AT THE DOOR: a user may not change an agent-authored action's kind or configuration. - - Three outcomes, and the middle one is the ruling: - * the action is ABSENT from the incoming flow -> a DELETE, and R9 allows it - * the action is present and CHANGED -> **409**, naming the action and the agent - * the action is present and identical -> nothing happens, so an ordinary save of a - flow that merely CONTAINS an agent action - is not refused - """ - marks = _agent_marks(session.runtime).get(str(auto_id)) or {} - if not isinstance(marks, dict) or not marks: - return - incoming = (body or {}).get("flow") - if not isinstance(incoming, dict) or "actions" not in incoming: - return - stored = (engine.all_definitions(session.runtime) or {}).get(str(auto_id)) or {} - was = {str(a.get("id") or ""): a for a in _flow_actions(stored)} - now = {str(a.get("id") or ""): a - for a in _flat_actions(incoming.get("actions") or [])} - for action_id, mark in marks.items(): - action_id = str(action_id) - if action_id not in now or action_id not in was: - continue # removed, or never stored: not an EDIT - if _same_action(was[action_id], now[action_id], session.runtime): - continue - who = str((mark or {}).get("agentName") or (mark or {}).get("agent") or "an agent") - raise err(409, "agent_owned", - f"this step was built by {who} and only {who} can change how it is set up. " - f"You can read it, and you can delete it, but it cannot be edited by hand") - - -def _prune_marks(session, auto_id): - """Drop annotations whose action is gone, and the whole entry when the automation is. - - ⚠ CALLED AFTER EVERY WRITE, because the alternative is an annotation bucket that only ever - grows: a mark on a deleted action would keep refusing an id nobody can see, and a mark on a - deleted automation would sit in the tenant document forever. - """ - auto_id = str(auto_id) - - def _set(cur): - cur = dict(cur or {}) - marks = cur.get(auto_id) - if not isinstance(marks, dict): - cur.pop(auto_id, None) - return cur - defn = (engine.all_definitions(session.runtime) or {}).get(auto_id) - if defn is None: - cur.pop(auto_id, None) - return cur - alive = {str(a.get("id") or "") for a in _flow_actions(defn)} - kept = {k: v for k, v in marks.items() if str(k) in alive} - if kept: - cur[auto_id] = kept - else: - cur.pop(auto_id, None) - return cur - - session.runtime.update(AGENT_ACTIONS_KEY, _set, flush="sync") - - -@router.post("/automations/{auto_id}/agent-actions") -def agent_author_action(auto_id: str, body: dict = Body(default=None), - session: Session = Depends(_GATE)): - """THE AGENT'S DOOR: add or replace one Action under the Canvas, marked agent-owned (R9). - - {agent: "", action: {kind, config, when?, id?}} - -> {automation, actionId, agent} - - ⛔ THE ACTION GOES THROUGH `clean_actions` LIKE EVERY OTHER ONE, and that is what "guardrail - the reach of this script" means in code: an agent cannot invent a config key, cannot name a - kind that is not in the catalog, and cannot reach a kind this tenant is not entitled to. The - agent gets a different WALL on editing, never a wider vocabulary. - - ⛔ AND IT CANNOT AUTHOR A KIND THAT IS NOT BUILT. `run_script` is `ready: False` in the - catalog, so `clean_actions` refuses it here exactly as it refuses it for a person — see this - lane's mailbox for why the script ARM is booked rather than half-built: a per-row script - contract and a client card are both missing, and a step that reports success and does nothing - is the failure this repo has already paid for. - - ⚠ THE AGENT ID IS CHECKED AGAINST THE TENANT'S OWN AGENTS. A mark naming an agent that does - not exist would refuse every future edit with a sentence naming nobody. - """ - import routes_slack # noqa: PLC0415 - - body = body if isinstance(body, dict) else {} - agent_id = str(body.get("agent") or "").strip() - agent = routes_slack._agents(session.runtime).get(agent_id) - if not isinstance(agent, dict): - raise err(404, "no_agent", "there is no agent with that id in this workspace") - action = body.get("action") - if not isinstance(action, dict) or not str(action.get("kind") or "").strip(): - raise err(400, "no_action", "an action needs a kind") - - defn = (engine.all_definitions(session.runtime) or {}).get(str(auto_id)) - if defn is None: - raise err(404, "unknown_automation", "no automation with that id") - if str(defn.get("system") or "").strip(): - raise err(409, "system_agent", engine.system_agent_refusal(str(defn["system"]))) - - existing = list(((defn.get("flow") or {}).get("actions") or [])) - taken = {str(a.get("id") or "") for a in _flat_actions(existing)} - action_id = str(action.get("id") or "").strip() - if not action_id or action_id not in taken: - n = 1 - while f"{_AGENT_ACTION_PREFIX}{n}" in taken: - n += 1 - action_id = f"{_AGENT_ACTION_PREFIX}{n}" - fresh = dict(action, id=action_id) - - # ⚠ VALIDATED BEFORE ANYTHING IS WRITTEN, so a refusal leaves the automation exactly as it was. - checked, error = engine.clean_actions([fresh], rt=session.runtime) - if error or not checked: - raise err(400, "invalid_action", error or "that action could not be built") - - replaced = False - for i, one in enumerate(existing): - if isinstance(one, dict) and str(one.get("id") or "") == action_id: - existing[i], replaced = fresh, True - break - if not replaced: - existing.append(fresh) - - notes = [] - updated, error = engine.patch(session.runtime, str(auto_id), - {"flow": {**(defn.get("flow") or {}), "actions": existing}}, - username=session.uname, notes=notes) - if error: - raise err(400, "invalid_automation", error) - - def _set(cur): - cur = dict(cur or {}) - marks = dict(cur.get(str(auto_id)) or {}) - marks[action_id] = {"agent": agent_id, - "agentName": str(agent.get("label") or agent.get("channelName") - or agent_id), - "created": datetime.now(timezone.utc).isoformat(timespec="seconds"), - "by": session.uname} - cur[str(auto_id)] = marks - return cur - - session.runtime.update(AGENT_ACTIONS_KEY, _set, flush="sync") - _prune_marks(session, auto_id) - return {"automation": _wire(updated, session.tenant, rt=session.runtime), - "actionId": action_id, "agent": agent_id, "notes": notes} - - -@router.get("/automations/{auto_id}") -def get_automation(auto_id: str, session: Session = Depends(_GATE)): - """One automation, by id. - - ⛔⛔ WAVE 34 · CONTRACT C3 — A SYNTHETIC ROW MUST BE FETCHABLE BY ID, NOT ONLY LISTABLE, AND - THIS IS THE HALF THAT IS EASY TO SHIP BROKEN. `list_automations` and this route are SEPARATE - lookups: a row appended only to the list appears in the rail, looks entirely real, and 404s - the instant somebody clicks it. `W34-T48`'s `done-when` is *"opening it shows a Trigger and - one step"* — i.e. this route, not the list — so the field agents are resolved here too, from - the same builder, and a person cannot reach a row the detail route does not know. - ⚠ THE SAME BUILDER, NOT A SECOND ONE. `_field_agent_rows` is called and the id looked up in - its output rather than re-deriving one row from the field: two derivations of one row is how - the list and the editor start disagreeing about a name. - """ - if str(auto_id) == ODOO_SYNC_ID: - # ⚠ `detail=True` — the keychain label and the last-sync stamp cost a store read each and - # are resolved HERE, on a route that fetches one automation, never on the list. - row = _odoo_sync_row(session, detail=True) - if row is None: - raise err(404, "unknown_automation", "no automation with that id") - return {"automation": row} - if str(auto_id).startswith("field:"): - row = next((r for r in _field_agent_rows(session) if r.get("id") == str(auto_id)), None) - if row is None: - # ⚠ The SAME 404 as any other unknown id. A field agent whose column has been deleted - # is genuinely gone, and inventing a different error for it would make a normal - # outcome look like a fault. - raise err(404, "unknown_automation", "no automation with that id") - return {"automation": row} - defn = engine.all_definitions(session.runtime).get(str(auto_id)) - if defn is None: - raise err(404, "unknown_automation", "no automation with that id") - return {"automation": _wire(defn, session.tenant, rt=session.runtime)} - - -@router.post("/automations") -def create_automation(body: dict = Body(default=None), session: Session = Depends(_GATE)): - """Create an automation. ⭐ WAVE 23 (C2): the body may carry - `target: {mode: "existing"|"new"|"automated", table?, label?}` — the wizard's FIRST question, - answered before the kind. `new` mints the blank database in the same call, so the automation - is never saved pointing at a table that does not exist yet.""" - if not session.runtime.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable. Nothing was saved") - # ⭐⭐ D-75 — THE DISCLOSURE CHANNEL, CONSUMED. `clean_actions` silently drops a - # `create_record` condition (D-65's law: dropping is recoverable, refusing locks the door) and - # any config key an arm's allowlist does not keep. Both are deliberate; what was missing was - # anybody being TOLD, so a person saved a flow and got a different one with nothing said. - # ⚠ It rides the SAVE's own response, beside `unconfigured`, because that is the moment the - # person is looking — a note in a log they never open is the same silence with extra steps. - _notes = [] - defn, error = engine.create(session.runtime, body or {}, username=session.uname, - notes=_notes) - if error: - raise err(400, "invalid_automation", error) - return {"automation": _wire(defn, session.tenant, rt=session.runtime), "notes": _notes} - - -@router.patch("/automations/{auto_id}") -def patch_automation(auto_id: str, body: dict = Body(default=None), - session: Session = Depends(_GATE)): - if not session.runtime.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable. Nothing was saved") - # ⛔⛔ WAVE 34 · CONTRACT C3 — A SYNTHETIC ROW IS NOT PATCHABLE, AND IT MUST SAY SO. A field - # agent is DERIVED from a column definition; there is nothing in the automations bucket to - # write. `engine.patch` would answer "no such automation" (a 404 that reads as "your agent - # vanished") or, worse for a future id shape, find nothing and report success. The refusal - # names where the setting actually lives, because a wall that does not say what to do instead - # sends somebody looking for a bug in a decision made on purpose (D-65's own lesson). - if str(auto_id).startswith("field:"): - raise err(409, "system_agent", - "this agent is an AI enrichment column. Change its prompt, model or schedule on " - "the column itself and this page follows") - if str(auto_id) == ODOO_SYNC_ID: - return _patch_odoo_sync(session, body or {}) - # D-75, same channel as `create` above — an EDIT is where this matters most, because the - # person has just typed the thing that gets dropped. - # ⛔⛔ W36-T38 / R9 — THE AGENT-OWNED WALL, AT THE DOOR, BEFORE ANY WRITE. A `disabled` - # attribute in the builder is a suggestion; this is the refusal. It permits a DELETE of the - # step and permits an ordinary save of a flow that merely contains one. - _agent_owned_guard(session, auto_id, body or {}) - _notes = [] - defn, error = engine.patch(session.runtime, auto_id, body or {}, username=session.uname, - notes=_notes) - if error: - raise err(400 if error != "no such automation" else 404, - "invalid_automation" if error != "no such automation" else "unknown_automation", - error) - # An action the user removed takes its annotation with it, or the mark outlives its subject - # and refuses an id nobody can see. - _prune_marks(session, auto_id) - return {"automation": _wire(defn, session.tenant, rt=session.runtime), "notes": _notes} - - -@router.delete("/automations/{auto_id}") -def delete_automation(auto_id: str, session: Session = Depends(_GATE)): - """Delete the DEFINITION. ⚠ The database it filled is NOT touched — an automation is the - thing that writes rows, not the thing that owns them, and deleting a job must never be a way - to lose data (the same rule the orphan count encodes). - - ⭐⭐ WAVE 34 · CONTRACTS C3 + C4 — A SYSTEM AGENT REFUSES, LOUDLY, AND THE REFUSAL SAYS WHERE - TO GO INSTEAD. Two families are undeletable and they are undeletable for different reasons, so - the sentence names the reason rather than saying "no": a FIELD AGENT is a column (delete the - column), and the ODOO SYNC is the connector's own schedule (disconnect the connector). - - ⛔ REFUSED, NOT IGNORED, AND THAT IS THE POINT OF THE FIRST BRANCH. A synthetic id is not in - the automations bucket, so `engine.remove` would find nothing, do nothing, and this route - would answer `200 {"deleted": ...}` — a delete that reports success and changes nothing, which - is the `view_upsert` failure mode (200 OK, zero writes) arriving in a new place. The row would - then reappear on the next poll and read as a bug in the rail. - """ - # ⛔⛔ EVERY SYNTHETIC ID, NOT JUST THE FIELD ONES. The first version of this guard listed - # `field:` alone, and `verify_automation`'s own C4 leg caught what that left open: deleting - # `system:odoo_sync` fell straight through to `engine.remove`, which popped a key that was - # never in the bucket and answered `200 {"deleted": ...}`. A delete that reports success and - # changes nothing, on the one row the ruling says must be undeletable — the exact defect the - # guard exists for, one id shape away. Both prefixes are refused by the same branch now. - if str(auto_id).startswith("field:"): - raise err(409, "system_agent", engine.system_agent_refusal(SYSTEM_FIELD_AGENT)) - if str(auto_id) == ODOO_SYNC_ID: - raise err(409, "system_agent", engine.system_agent_refusal(SYSTEM_ODOO_SYNC)) - if engine.running(session.tenant, auto_id): - raise err(409, "automation_running", "it is running. Wait for it to finish") - # ⚠ READ BEFORE THE REMOVE, not after: `engine.remove` is the thing being refused. - _defn = (engine.all_definitions(session.runtime) or {}).get(str(auto_id)) or {} - _sys = str(_defn.get("system") or "").strip() - if _sys: - raise err(409, "system_agent", engine.system_agent_refusal(_sys)) - engine.remove(session.runtime, auto_id) - # R9's other half: the user MAY delete an agent-authored step, and deleting the whole - # automation takes its annotations with it rather than stranding them in the document. - _prune_marks(session, auto_id) - return {"deleted": str(auto_id)} - - -@router.post("/automations/preview") -def preview_source(body: dict = Body(default=None), session: Session = Depends(_GATE)): - """What does this URL actually offer? The field-map step — never writes anything.""" - body = body or {} - url = str(body.get("url") or "").strip() - if not url: - raise err(400, "no_url", "give a page URL to read") - try: - return engine.preview(url, str(body.get("extract") or "table"), - int(body.get("tableIndex") or 0)) - except engine.Refused as e: - raise err(400, "refused_url", str(e)) - except Exception as e: # noqa: BLE001 - raise err(502, "fetch_failed", f"could not read that page. {type(e).__name__}: " - f"{str(e)[:160]}") - - -@router.post("/automations/{auto_id}/run") -def run_automation(auto_id: str, session: Session = Depends(_GATE)): - """Start a run on a background thread. 409 when one is already in flight.""" - defn = engine.all_definitions(session.runtime).get(str(auto_id)) - if defn is None: - raise err(404, "unknown_automation", "no automation with that id") - # ⛔ WAVE 32 · T45 (owner item 10) — REFUSED, NAMING THE ACTION. `run_now` refuses too, for the - # tick and the webhook; this one exists so the person who pressed the button reads the reason - # instead of watching a run start and end with nothing done. The two ask the SAME function, so - # they cannot come to disagree about what "configured" means. - refusal = engine.run_refusal(defn) - if refusal: - raise err(400, "action_unconfigured", refusal) - if not engine.run_async(session.runtime, session.tenant, auto_id, username=session.uname): - raise err(409, "automation_running", "that automation is already running") - return {"started": str(auto_id), "startedAt": time.strftime("%Y-%m-%dT%H:%M:%S")} - - -@router.post("/automations/{auto_id}/nodes/{node_id}/toggle") -def toggle_automation_node(auto_id: str, node_id: str, session: Session = Depends(_GATE)): - """Flip one step on the canvas. The SERVER decides what a node's switch means (see - `engine.NODE_TOGGLES`) — the client only reports which node was clicked. - - A node with no switch answers 400 with the sentence saying why, rather than silently doing - nothing: a control that appears to work and does not is worse than one that refuses. - """ - if not session.runtime.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable. Nothing was saved") - if engine.running(session.tenant, auto_id): - raise err(409, "automation_running", "it is running. Wait for it to finish") - defn, error = engine.toggle_node(session.runtime, auto_id, node_id) - if error: - raise err(404 if error == "no such automation" else 400, - "unknown_automation" if error == "no such automation" else "node_not_toggleable", - error) - return {"automation": _wire(defn, session.tenant, rt=session.runtime)} - - -@router.post("/automations/{auto_id}/hook/{token}") -async def automation_hook(auto_id: str, token: str, request: Request): - """C3's webhook trigger — the tick-endpoint pattern one level down: unauthenticated BY - DESIGN (the external caller has no cookie), gated on a per-automation token minted when the - trigger was configured, constant-time compared. The tenant is FOUND by the (id, token) - pair — a wrong token answers 403 for every tenant, so the route confirms nothing about - which slugs exist. - - ⭐ WAVE 24 (D-41) — THE BODY IS READ DEFENSIVELY AND IS NEVER A REASON TO REFUSE. - ⛔ It is deliberately NOT declared as `body: dict = Body(...)`, which is the obvious way to - write this and would be a live regression: FastAPI would then VALIDATE the payload, so an - existing caller posting text, form-encoding, an empty body or slightly malformed JSON would - start getting a 422 from a door that has accepted anything since wave 22. A webhook sender is - somebody else's system; we do not get to change what it must send in order to fire a flow. - An unreadable body simply maps nothing — the flow still fires, exactly as it did before. - - `run_in_threadpool` keeps the store I/O off the event loop: `hook_fire` walks every tenant - and may commit a row, and this handler had to become `async` only to read the request body. - """ - from starlette.concurrency import run_in_threadpool - from harness import runtime as _rt - - try: - payload_in = await request.json() - except Exception: # noqa: BLE001 - payload_in = None - - def _fire(): - last = (404, {"error": "unknown_automation", - "message": "no automation with that id and token"}) - for slug in _rt.known_tenants(): - try: - rt = _rt.get_runtime(slug) - except Exception: # noqa: BLE001 - continue - status, payload = engine.hook_fire(rt, slug, auto_id, token, body=payload_in) - if status == 200: - return 200, payload - if status != 404: - last = (status, payload) - return last - - status, payload = await run_in_threadpool(_fire) - if status == 200: - return payload - raise err(status, str(payload.get("error") or "refused"), - str(payload.get("message") or "refused")) - - -@router.post("/automations/tick") -def tick(request: Request, x_aios_tick_token: str = Header(default="")): - """Fire every due schedule, for every tenant. The durable-cron entry point (R5). - - Unauthenticated BY DESIGN and gated on a shared secret instead — an EventBridge rule has no - cookie. Refuses when the secret is not configured (see the module header).""" - want = os.environ.get("AIOS_AUTOMATION_TICK_TOKEN") or "" - if not want: - raise err(403, "tick_disabled", - "AIOS_AUTOMATION_TICK_TOKEN is not configured. The tick endpoint is closed") - got = x_aios_tick_token or request.headers.get("X-AIOS-TICK-TOKEN") or "" - if got != want: - raise err(403, "bad_tick_token", "that token is not valid for this deployment") - started = engine.tick_all() - return {"started": started, "at": time.strftime("%Y-%m-%dT%H:%M:%S")} - - -@router.post("/automations/ig/purge") -def purge_ig_subject(body: dict = Body(default=None), session: Session = Depends(_GATE)): - """D-24 (wave 22): the right-to-erasure door for ONE Instagram subject — walks the - tenant's four `ut_ig_*` tables AND the platform master (R2 pooled a copy there, so a purge - that skipped it would not be erasure). Admin-only: erasure is a compliance act, not a - grid gesture. Answers the per-table removal counts — the one aggregate whose drill is the - rows' ABSENCE.""" - if not session.admin: - raise err(403, "admin_only", "erasing a subject is an admin action") - if not session.runtime.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable. Nothing was " - "purged") - handle = str((body or {}).get("handle") or "").strip() - if not handle: - raise err(400, "no_handle", "name the Instagram handle to erase") - counts = engine.purge_subject(session.runtime, handle) - return {"handle": handle.lstrip("@").lower(), "removed": counts, - "total": sum(counts.values())} - - -@router.get("/automations/metrics/{table_key}/{field_key}/{row_id}/rows") -def metric_drill(table_key: str, field_key: str, row_id: str, - session: Session = Depends(_GATE)): - """C7's drill: the EXACT master snapshot rows behind one metric cell - ([[no-unverifiable-aggregates]]) — recomputed on ask with the same function that filled - the cell, so the drill can never disagree with the number by construction.""" - import core.user_tables as user_tables - - if not user_tables.may_open(table_key, session.uname, session.admin, st=session.runtime): - raise err(404, "unknown_table", "no such database") - t = engine.ut_get(session.runtime, table_key) or {} - fdef = next((f for f in (t.get("fields") or []) if f.get("key") == field_key), None) - if not fdef or not isinstance(fdef.get("metric"), dict): - raise err(404, "not_a_metric", "that column is not a metric field") - row = (t.get("rows") or {}).get(str(row_id)) - if row is None: - raise err(404, "unknown_row", "that record is not in the database") - import ig_master - url_field = next((f.get("key") for f in (t.get("fields") or []) - if f.get("type") == "url"), "") - handle = engine._table_handle(row, url_field) - bag = fdef["metric"] - series = ig_master.series_for({handle}) if handle else {} - value, rows = engine.metric_value(series.get(handle), bag.get("measure"), - bag.get("window"), bag.get("agg") or "") - return {"table": table_key, "field": field_key, "rowId": str(row_id), "handle": handle, - "measure": bag.get("measure"), "window": bag.get("window"), - "value": value, "rows": rows[:200], - "note": "" if value is not None else - "no master data answers this window. The cell is honestly blank"} - - -@router.get("/automations/{auto_id}/rows") -def run_rows(auto_id: str, session: Session = Depends(_GATE)): - """The rows the LAST run touched — the drill-down behind a run's counts. - - Every count in this product drills to the exact rows behind it ([[no-unverifiable-aggregates]]); - a run history that said "412 updated" and could not show which would be the thing that rule - exists to forbid. - """ - defn = engine.all_definitions(session.runtime).get(str(auto_id)) - if defn is None: - raise err(404, "unknown_automation", "no automation with that id") - last = (defn.get("runs") or [{}])[0] - table_key = (defn.get("config") or {}).get("targetTable") or "" - t = engine.ut_get(session.runtime, table_key) or {} - rows = t.get("rows") or {} - ids = [str(i) for i in (last.get("affected") or [])] - return {"table": table_key, "label": t.get("label") or table_key, - "fields": [{"key": f.get("key"), "label": f.get("label")} - for f in (t.get("fields") or [])], - "rows": [{"id": i, **(rows.get(i) or {})} for i in ids if i in rows], - "truncated": len(ids) >= 200} - - -# --- the in-process scheduler ------------------------------------------------------------ -# ⚠ OPT-IN (`AIOS_AUTOMATIONS=1`), and that is an AMENDMENT to the wave brief's "default-on", -# made on a measurement: `verify_api.py` runs for 196 s, i.e. longer than three tick intervals, -# and `tick_all` reaches `runtime.get_runtime()` — which builds tenants and moves the LRU cache -# that verify_api's own isolation checks read. Default-on would put a background thread inside -# the subject of another session's gate. `AIOS_PREWARM=1` at `main.py:353` is the same decision -# for the same reason, so this follows it rather than inventing a second convention. -# -# The loop also SLEEPS FIRST (`engine.scheduler_loop`) — defence in depth, proven in -# verify_automation.py section T rather than assumed. -# -# ⛔ THE DEPLOY MUST SET `AIOS_AUTOMATIONS=1` (with `AIOS_AUTOMATION_TICK_TOKEN`) or schedules -# only ever fire from the external cron POSTing /automations/tick. Both paths work; neither is -# implicit. Booked in the session-D mailbox. -engine.start_scheduler() - -# C3 (wave 22): register the trigger listener onto the platform's row-event seam. THIS module -# is where the registration belongs — it is the one place that already imports both sides, so -# neither the engine nor platform/core grows a dependency on the other. Idempotent: a reimport -# must not double-fire every trigger. -import core.user_tables as _ut_hooks # noqa: E402 - -if engine.grid_hook not in _ut_hooks.ROW_HOOKS: - _ut_hooks.ROW_HOOKS.append(engine.grid_hook) - -# ⭐⭐ W31 QA — DECLARE THE MACHINE-OWNED CHILD DATABASES, for the same reason and in the same -# place as the hook above: this module already imports both sides, so neither the engine nor -# `platform/core` grows a dependency on the other. Idempotent by construction (a set). -# -# ⛔ THE OWNER FOUND WHAT THIS FIXES, IN PRODUCTION, AFTER THE TICKET READ GREEN. W31-T32 locked -# the TikTok children by stamping `recordMode` at the two SPAWN sites, and proved the stamp -# arrives "on the next write". That is true and it is not the `done-when`: every TikTok database -# already sitting in a tenant kept offering "+ New record" until somebody re-ran a TikTok -# automation, and nobody had. Owner, verbatim (2026-08-13): *"the databases for Tiktok do not have -# the small lock icon as I asked"* — and the rule, restated: *"just like Instagram Post database -# (which is locked), only the IG Profile and TT Profile should be editable."* -# ⚠ A DECLARATION NEEDS NO WRITE, so it is true for EVERY tenant the moment the API boots — no -# migration, no boot ordering, no per-tenant walk, and nothing that a stale store can undo. The -# stored flag still locks a table nobody declares; the two are OR'd. -_ut_hooks.register_locked_records(engine.LOCKED_CHILD_TABLES) +"""routes_automation.py — wave-18 item 5 (contract C4-AUTO): the automation surface's API. + +The router is thin on purpose: everything that can be wrong about an automation — a malformed +cron, a URL the SSRF rail refuses, a key field that is not one of the mapped columns — is decided +in `automation_engine`, which is a pure-ish module a gate can drive without a server. This file +does auth, shape and status codes. + +⛔ THE TICK ENDPOINT IS THE ONE UNAUTHENTICATED ROUTE, AND IT IS FAIL-CLOSED TWICE OVER. It takes +no session (an external cron has no cookie), so it is gated on a shared secret in +`X-AIOS-TICK-TOKEN`; and when `AIOS_AUTOMATION_TICK_TOKEN` is UNSET the route refuses everything +rather than admitting everyone. A "no token configured means no check" default is how an internal +trigger becomes a public one — the same class of mistake as an empty-200 permission answer. +""" +import os +import re +import time +from datetime import datetime, timezone + +from fastapi import APIRouter, Body, Depends, Header, Request + +import ai_review +import automation_engine as engine +import oauth_connect +import routes_oauth +import scope_cache +from deps import Session, err, module_gate, require_session + +router = APIRouter(prefix="/api/v1") + +# C5 (wave 22): the OAuth connector surface rides INSIDE this router — main.py belongs to no +# session this wave, and this router is already mounted there. `/api/v1` + `/oauth/...`. +router.include_router(routes_oauth.router) +# ⛔⛔ D-203 — THE NESTED INCLUDE OF `routes_connectors` IS REMOVED, and the reason it existed is +# worth keeping because it was a GOOD reason that expired. +# +# Wave 23 (C11) mounted the connectors directory here rather than in `main.py`, because this router +# was already mounted and `main.py` belonged to another session — a sensible way to avoid a +# cross-fence ask. `main.py:227` includes `routes_connectors` DIRECTLY now, so this line stopped +# being the only door and became a SECOND one. +# ⚠ AND A ROUTER MOUNTED TWICE DOES NOT SERVE THE SAME PATHS TWICE — it serves the prefix twice. +# `routes_connectors` carries its own `/api/v1`, so nesting it inside this router's `/api/v1` +# produced **`/api/v1/api/v1/connectors/directory`**: a live, session-gated, entirely dead path +# that nothing links to and every route audit has to explain. MEASURED before removal: 123 served +# paths, exactly 1 of them doubled. +# ⚠ `routes_oauth` above is NOT the same case and stays: `main.py` does not mount it, so this +# router is genuinely its only door. Deleting it because its neighbour was wrong is how a real +# route dies for a tidy-up. +import routes_connectors # noqa: E402,F401 + +#: The registry key this surface carries (C-AUTONAV — A adds the row; the gate is live now, so +#: the day the row lands the wall is already the one that was tested). +MODULE = "automation" + +_GATE = module_gate(MODULE) + + +def _wire(defn, tenant, rt=None): + """One automation, as the client reads it. `running` is PROCESS state, never store state — + see the engine header on why a persisted 'running' is a permanent lock.""" + live = engine.running(tenant, defn.get("id")) + sched = defn.get("schedule") or {} + nxt = engine.next_fire(sched.get("cron")) if sched.get("enabled") else None + status = dict(defn.get("status") or {}) + if live: + status = {**status, "state": "running", "startedAt": live.get("startedAt"), + "step": live.get("step")} + return { + "id": defn.get("id"), "name": defn.get("name"), "kind": defn.get("kind"), + "config": defn.get("config") or {}, "schedule": sched, "status": status, + "runs": list(defn.get("runs") or [])[:engine.MAX_RUNS], + "created": defn.get("created"), "createdBy": defn.get("createdBy"), + "nextRunAt": nxt.strftime("%Y-%m-%d %H:%M") if nxt else "", + "running": bool(live), + # ⭐ WAVE 27 (contract C6) — DEBT D-70: "a paid search is already outstanding at the + # vendor". THE MONEY GUARD, and it was INERT for a whole wave because this line was + # missing: the client declared `awaitingResults` REQUIRED and read it in `runBlock()`, + # nothing ever sent it, so `undefined` was falsy and the Run button stayed armed. + # + # ⛔ WHY IT IS NOT `running`. `running` above is PROCESS state (`engine.running`) and is + # FALSE for the entire 20–30 MINUTE vendor wait, which IS the hazard window: the ticks + # are 3 minutes apart, so between them the automation is idle and the button re-arms. + # MEASURED the expensive way on 2026-08-06 (~$0.025): with no snapshot outstanding a + # Run-now press returned 200 and started a BRAND-NEW billable corpus search. + # + # ⛔ AND NOT A PERSISTED "running" FLAG EITHER — the engine header forbids one (it + # outlives the process and locks the automation forever). `state.pendingSnapshot` is the + # field that already exists, is already persisted, is already carried untouched through + # `clean_definition`, and is the SAME expression `pending_collect_ids()` uses to decide + # what to collect. One truth, two readers. + "awaitingResults": bool(str((defn.get("state") or {}).get("pendingSnapshot") + or "").strip()), + # C3 (wave 22): the trigger config rides whole — the webhook token included, because + # the person configuring the external caller has to be shown the URL somewhere, and + # this payload is session-gated behind the same wall as everything else here. + # ⭐⭐ WAVE 35 · T36 / R10 — THE PARKED BATCH, SUMMARISED. Without this line the whole + # ticket would be unreachable: `_wire` is a WHITELIST, so a key the engine parks on the + # definition simply never arrives, and the detail pane could not know a batch was waiting + # ([[reachable-is-not-the-same-as-built]] — the exact shape `flow` was caught in above). + # ⚠ COUNT AND NOTES ONLY, NEVER `items`. A 200-statement batch carries 200 rendered HTML + # mails; this payload rides on the automations LIST, which every rail paint reads. + # `GET /admin/statements/agent/{id}` serves the detail when somebody opens the batch. + "pendingStatements": ({ + "count": int((defn.get("pendingStatements") or {}).get("count") or 0), + "ts": (defn.get("pendingStatements") or {}).get("ts") or "", + "notes": list((defn.get("pendingStatements") or {}).get("notes") or [])[:25], + } if isinstance(defn.get("pendingStatements"), dict) else None), + # ⭐⭐ WAVE 35 · T37 — `system` FOR A **STORED** ROW. Until now only SYNTHETIC rows carried + # it, stamped by their builders AFTER this function (`_odoo_sync_row`, `_field_agent_rows`), + # so a stored row with the marker had it enforced on the server (`delete_automation` 409s) + # and INVISIBLE to the client — which paints a live-looking Delete that answers with a + # refusal. `AutomationDetail` already reads `automation.system` to disable that button and + # explain why; this is the line that lets it. + # ⚠ Absent stays absent rather than becoming `""`: every ordinary automation is not-a-system + # -agent, and an empty string is a value a client could accidentally treat as one. + **({"system": str(defn.get("system") or "")} if defn.get("system") else {}), + "trigger": defn.get("trigger") or None, + "statusNote": defn.get("statusNote") or "", + # The one-sentence summary (airtable-brief rec 7), composed from the definition so it + # cannot describe steps the engine does not run. + "sentence": engine.compose_sentence(defn), + # THE CANVAS TOPOLOGY (R5) rides with the automation rather than being rebuilt in the + # client, for the same reason `cronPresets` does: the engine that RUNS the steps is the + # only thing entitled to say what the steps are. + "graph": engine.graph(defn), + # ⭐ WAVE 23 (C4/C5) — THE BUILDER'S OWN STATE, and its absence here was a silent-drop + # bug caught in review rather than by a gate: `flow` was stored, patchable and validated, + # but never sent back. B would have saved a flow through PATCH, got a 200, and watched + # every action vanish on reload — the classic "it didn't save" with nothing red anywhere. + "flow": defn.get("flow") or {"actions": []}, + # ⭐⭐ WAVE 32 · T45 (owner item 10) — CONFIGURED / UNCONFIGURED, per action, on the wire. + # + # ⛔ THE SERVER SAYS IT, not the client, and that is the whole reason it is here: the same + # `engine.action_needs` answers this label, `engine.run_refusal`'s 400 and `run_now`'s own + # refusal, so a card cannot read "Configured" over an action the run will refuse. Three + # readers, one predicate — the alternative is a client-side rule that agrees with the + # server until somebody adds a key to one of them (`awaitingResults` above is this file's + # own record of what the other shape costs). + # ⚠ IT IS A LIST, KEYED BY ACTION ID, and it names the NESTED actions too — an unconfigured + # step inside an If / then branch is exactly the one a person cannot see. + # ⚠ COSTS NO STORE READ. `action_needs` is pure over `(kind, config)`; this route is the + # one W30-T12 took the `user_tables` deep copy off, and a label is not worth putting it + # back. What needs the target database's schema (an enrich binding resolved by the profile + # FLAG) stays a run-time refusal — see `ACTION_REQUIRED`'s note. + "unconfigured": engine.unconfigured_actions(defn), + # ⭐⭐ WAVE 36 · W36-T38 (owner item 8 / R9) — WHICH ACTIONS AN AGENT BUILT. + # `{action_id: {agent, agentName, created, by}}`, empty for an ordinary automation. + # ⛔ THE CONFIG IS NOT HIDDEN AND MUST NOT BE. R9 is a WRITE rule: the client uses this to + # draw a lock and explain who owns the step, never to withhold what the step does — an + # automation nobody can audit is worse than one nobody can edit. + # ⚠ `rt=None` YIELDS `{}` RATHER THAN OMITTING THE KEY, so a caller that forgets the + # runtime produces "nothing is agent-owned" (a visible, wrong-but-safe answer) instead of + # an absent prop the client silently reads as undefined + # [[flag-shipped-without-its-writer]]. + "agentActions": (_agent_marks(rt).get(str(defn.get("id") or "")) or {}) if rt else {}, + } + + +#: ⭐⭐ WAVE 34 · CONTRACTS C3 + C4 — WHY A ROW CANNOT BE DELETED, as a slug. +#: +#: ⚠ ALWAYS A STRING, NEVER A BOOLEAN, and the two contracts disagreed about that (C3 says +#: `system: true`, C4 says `system: ""`). Raised as ask `E-13`; built as a slug because one +#: key with two types is [[one-question-two-normalizers]] before a line is written, and because a +#: refusal has to be able to NAME the reason. Truthiness is unchanged for any reader that only +#: asks "is this a system agent". +SYSTEM_FIELD_AGENT = "field_agent" +SYSTEM_ODOO_SYNC = "odoo_sync" + + +def _field_agent_rows(session): + """⭐⭐ CONTRACT C3 — one synthetic Agent row per `ai_enrich` column in the tenant. + + R13: *"Every field agent must be VIEWABLE under the Agent module as a simple Trigger whose + first step is the field enrichment, with the Field and Database shown under Config."* + + ⛔⛔ THESE ARE DERIVED, NOT STORED, AND NOTHING HERE MAY WRITE. A field agent's only source of + truth is the column definition F's `_clean_field` stamps; a copy in the automations bucket + would be a second one, and the first PATCH through the enrichment editor would silently + diverge from it. `delete_automation` refuses these ids for the same reason: accepting a delete + that cannot delete anything is the `view_upsert` failure mode (200 OK, nothing written) in a + new place. + + ⛔⛔ AND IT USES THE ROWS-FREE PROJECTION, WHICH IS THE WHOLE ENGINEERING PROBLEM OF THIS + TICKET. `GET /automations` is the route W29-T01 and W30-T12 spent two waves taking the + whole-document read OFF: it reads `user_tables` exactly ONCE PER TENANT PER PROCESS now, and + this file's own `unconfigured` note says *"a label is not worth putting it back"*. Walking + every table's fields for `ai_enrich` columns is exactly that read. So this uses + `user_tables.all_defs` — the same projection `/nav` uses, MEASURED there at 1,750 ms -> 1.4 ms, + because tenant #0's document is 99.89% rows and a field definition is not a row. + ⚠ `all_defs` RAISES on `rows` rather than answering `{}`. Nothing here wants a row; if that + ever changes, the error names the projection instead of painting an empty grid. + + ⚠ THE PERMISSION WALL IS CALLED, NEVER RE-IMPLEMENTED. `may_open` decides which databases this + caller may see, lent the projected document exactly as `/nav` lends it. This route's own + `automation_tables` carries the scar that makes that non-negotiable: it once re-implemented + the wall, got it WIDER, and showed people databases they were refused the moment they clicked. + """ + import core.user_tables as user_tables + + try: + defs = user_tables.all_defs(st=session.runtime) or {} + lent = user_tables.lend(session.runtime, **{user_tables.STORE_KEY: defs}) + except Exception: # noqa: BLE001 + # A store blip must not take the whole rail down with it. The stored automations are the + # payload's real subject; field agents are an addition to it. + return [] + + out = [] + for key, defn in sorted(defs.items(), key=lambda kv: (kv[1].get("label") or "").lower()): + if not user_tables.may_open(key, session.uname, session.admin, st=lent): + continue + table_label = defn.get("label") or key + for f in user_tables.ai_enrich_fields(defn): + bag = f.get("automation") if isinstance(f.get("automation"), dict) else {} + if bag.get("kind") != SYSTEM_FIELD_AGENT: + continue + col = str(f.get("key") or "") + if not col: + continue + # ⚠ THE ID CARRIES THE TABLE AND THE COLUMN, and `table` is NOT read off the bag: + # F's `_clean_field` is handed ONE field dict with no table context, so it cannot + # stamp one (mailbox `F-3`). E holds that key because E is the side that iterates + # tables. Two sources for one fact was the alternative, and it drifts. + trig = bag.get("trigger") if isinstance(bag.get("trigger"), dict) else {} + mode = str(trig.get("mode") or "manual") + cron = str(trig.get("cron") or "").strip() + defn_syn = { + "id": f"field:{key}:{col}", + "name": f.get("label") or col, + "kind": SYSTEM_FIELD_AGENT, + # ⛔⛔ TWO TRIGGER VOCABULARIES MEET HERE AND THEY ARE NOT THE SAME ONE. A field + # agent's trigger is the ENRICHMENT vocabulary (`{mode, cron}`, values + # manual/on_change/schedule); an automation's is the ENGINE's (`{key}` plus a + # separate `schedule {cron, enabled}`). Handing the engine's shape a `mode` it + # cannot read is not a type error — it is SILENT: `compose_sentence` fell back to + # "When you press Run now" for a column scheduled at 06:00, and `graph()` drew a + # trigger node with no trigger. Measured before this mapping existed, which is the + # only reason it does. + # ⚠ THE MAP IS EXACT WHERE AN EQUIVALENT EXISTS AND HONEST WHERE IT DOES NOT: + # `manual` and `schedule` are the engine's own keys, and `on_change` becomes + # `event_field` ("When a record matches conditions"), the nearest thing the engine + # has to "a cell this column reads has moved". `mode` is kept verbatim beside it so + # nothing is lost in translation and the enrichment editor stays the source. + "trigger": {"key": {"manual": "manual", "schedule": "schedule", + "on_change": "event_field"}.get(mode, "manual"), + "mode": mode, **({"cron": cron} if cron else {})}, + **({"schedule": {"cron": cron, "enabled": True}} + if mode == "schedule" and cron else {}), + "flow": {"actions": [{"id": "act_1", "kind": "ai_enrich", + "config": {"table": key, "field": col, + "tableLabel": table_label}}]}, + } + # ⭐ THROUGH `_wire`, NOT BESIDE IT. A synthetic row hand-built to "look like" a + # stored one is a second shape that agrees until somebody adds a key to the real one + # — the exact way `awaitingResults` shipped inert for a whole wave. Passing the + # synthetic DEFINITION through the same function makes them identical by + # construction; only `system` is stamped afterwards, because no stored row has it. + row = _wire(defn_syn, session.tenant, rt=session.runtime) + row["system"] = SYSTEM_FIELD_AGENT + out.append(row) + return out + + +#: ⭐⭐ WAVE 34 · W34-T47 (mailbox D-5) — R6 APPLIED TO WORDS A MODEL WROTE. +#: +#: ⛔ `web_prose` READS FILES, so it is structurally blind to a dash that arrives at RUN TIME. +#: `POST /automations/draft` returns three strings a language model authored (the flow's name, +#: each step's `why`, and the dropped/notes sentences) and `W34-T42` is what put them on a screen. +#: Lane D MEASURED that a prompt instruction does not hold: their system prompt ends "Never use an +#: em dash" and the very next live cerebras turn came back with one. So this is code at the +#: boundary, not a better prompt. +#: ⚠ A DIGIT RANGE IS A DIFFERENT SENTENCE and gets the first rule: "10-20" means "10 to 20", and +#: turning it into "10, 20" states two numbers where the model stated a span. +_DRAFT_DASH = "[" + chr(0x2014) + chr(0x2013) + "]" + + +def _draft_no_dashes(text): + """Model prose with no em or en dash, and no meaning changed on the way.""" + text = str(text or "") + text = re.sub(rf"(?<=\d)\s*{_DRAFT_DASH}\s*(?=\d)", " to ", text) + text = re.sub(rf"\s*{_DRAFT_DASH}\s*(?=[,.;:!?])", "", text) + text = re.sub(rf"(?<=[,;:])\s*{_DRAFT_DASH}\s*", " ", text) + return re.sub(rf"\s*{_DRAFT_DASH}\s*", ", ", text) + + +ODOO_SYNC_ID = "system:odoo_sync" +#: The cadence presets in the words a person reads. ⚠ Keyed on `odoo_relational.SYNC_PRESETS`, and +#: a key with no phrase here falls back to "Every " rather than raising — a missing caption +#: must degrade to something readable, not take the agents list down. +ODOO_CADENCE_PHRASE = {"30m": "Every 30 minutes", "1h": "Every hour", + "4h": "Every 4 hours", "daily": "Every day"} + + +def _odoo_sync_row(session, detail=False): + """⭐⭐ CONTRACT C4 (ruling R22) — the Odoo sync, as an agent you can SEE. + + R22: *"Implicit time-triggered work (Odoo syncing) becomes EXPLICIT: a pre-set agent that + cannot be deleted, whose Canvas shows Trigger = time and Action = data syncing from Odoo, with + the keychain/API configuration shown."* + + ⛔⛔ DERIVED, NOT STORED, AND C4 SAYS "a stored automation may carry `system:`". THE MECHANISM + C4 ASKS FOR IS BUILT (`delete_automation` refuses any stored row carrying `system`); this + particular agent does not use it, for three measured reasons, and the deviation is raised as + ask `E-15` rather than taken quietly: + 1. NOTHING CAN SEED IT. There is no system-automation seeding path anywhere, and `D-195` + measured three times that a CLI write to the tenant store while the Space is live REPORTS + SUCCESS and is reverted within a minute. A stored row would have to be minted by the + container, i.e. a write-on-read on the busiest list route in the product. + 2. STORED MEANS RUNNABLE BY MACHINERY BUILT FOR USER AUTOMATIONS. If its runs went through + `_commit_run`, `CONSECUTIVE_FAILURE_PAUSE` would flip `schedule.enabled` off after K bad + passes — silently disabling the tenant's REAL Odoo cadence, not a cosmetic card. + 3. TWO COPIES OF ONE CADENCE. The loop reads `odoo_relational.sync_seconds`; a stored + automation would carry its own `schedule.cron`, and C4's own requirement is that the card + "cannot drift from the running loop". Deriving makes drift impossible instead of + forbidden. + + ⚠ `detail=False` IS THE LIST PATH AND IT STAYS CHEAP. The keychain label and the last-sync + stamp each cost their own store read, and `GET /automations` is the route two waves were spent + taking reads OFF. They are resolved only on the DETAIL fetch, which is one automation and one + click. Returns `None` when this tenant has no Odoo at all — an agent for a connector nobody + connected is a card that lies. + """ + try: + import odoo_relational as rel + except Exception: # noqa: BLE001 + return None + try: + cfg = rel.read_config(session.runtime) + secs = rel.sync_seconds(session.runtime) + frozen = bool(rel.frozen(session.runtime)) + except Exception: # noqa: BLE001 + return None + + every = str(cfg.get("syncEvery") or rel.DEFAULT_SYNC) + manual = secs is None + defn = { + "id": ODOO_SYNC_ID, + "name": "Odoo sync", + "kind": SYSTEM_ODOO_SYNC, + # ⚠ THE CADENCE IS READ, NEVER STORED HERE. `everySeconds` is what the loop will actually + # sleep for, so the card cannot claim a schedule the loop is not keeping. + "trigger": {"key": "manual" if manual else "schedule", "preset": every, + "everySeconds": secs, "presets": sorted(rel.SYNC_PRESETS)}, + **({} if manual else {"schedule": {"cron": "", "enabled": True}}), + "flow": {"actions": [{"id": "act_1", "kind": "odoo_sync", + "config": {"connector": "odoo", "every": every, + "frozen": frozen}}]}, + } + row = _wire(defn, session.tenant, rt=session.runtime) + row["system"] = SYSTEM_ODOO_SYNC + # ⛔ THE SENTENCE IS OVERRIDDEN, AND IT IS A FIX RATHER THAN A PREFERENCE. `compose_sentence` + # speaks the automation vocabulary — it builds "When , run N actions" out of + # `schedule.cron` — and this agent has no cron: its cadence is a PRESET in seconds that the + # resync loop sleeps on. MEASURED before this line existed, the card read exactly + # `", run 1 action."`: a leading comma where the trigger phrase should have been. Two + # vocabularies again, and this one fails in punctuation rather than in behaviour, which is + # why only reading the output catches it. + # ⚠ AND THE CADENCE IS SPELLED OUT RATHER THAN INTERPOLATED. `f"Every {every}"` produced + # "Every daily, sync data from Odoo." — the preset KEYS are storage tokens ("30m", "daily"), + # not English, and a sentence built by pasting one in is only accidentally readable for the + # three that happen to be durations. Caught by reading the output, not by any assertion. + row["sentence"] = ("Odoo is disconnected, so nothing syncs." if frozen else + "Manually, sync data from Odoo." if manual else + f"{ODOO_CADENCE_PHRASE.get(every, f'Every {every}')}, sync data from Odoo.") + # ⛔ R6's SECOND SENTENCE, WHICH IS THE HALF THAT GETS DROPPED: a limit that cannot be removed + # must be REPORTED with its cause. Two are reported here rather than left for somebody to + # discover by watching a mirror not move. + notes = [] + if frozen: + notes.append("Odoo is disconnected, so this agent is not syncing. Reconnect it in " + "Connectors.") + if manual: + notes.append("The cadence is set to manual, so nothing syncs on a timer.") + row["statusNote"] = " ".join(notes) + if not detail: + return row + + # ── the DETAIL half: two store reads, on a route that fetches one automation ────────────── + try: + import routes_keychain as kc_routes # noqa: PLC0415 + row["flow"]["actions"][0]["config"]["keychain"] = ( + kc_routes.odoo_config(session).get("label") or "Odoo") + except Exception: # noqa: BLE001 + # A label is a nicety; failing to read one must not make the agent unopenable. + row["flow"]["actions"][0]["config"]["keychain"] = "Odoo" + try: + from harness import datastore as ds # noqa: PLC0415 + # ⭐ THE LAST RUN IS THE MIRROR'S OWN `_sync_state`, not a run history we would have to + # write. That table already carries a real per-entity `updated` stamp, so the card reports + # what actually happened rather than what this module remembers happening. + st = ds.status() or {} + stamps = sorted(str((v or {}).get("updated") or "") for v in st.values() if v) + last = [s for s in stamps if s] + if last: + row["status"] = {**(row.get("status") or {}), "state": "ok", "lastRunAt": last[-1], + "lastSummary": f"{len(last)} datasets synced"} + except Exception: # noqa: BLE001 + pass + return row + + +def _patch_odoo_sync(session, body): + """⭐⭐ CONTRACT C4 — THE ONE THING ON THIS CARD THAT IS EDITABLE: how often it runs. + + C4: *"Its Canvas is READ-ONLY except `trigger.config.everySeconds`, which writes through to the + same value `odoo_relational.sync_seconds` reads, so the card cannot drift from the running + loop."* + + ⛔ IT WRITES THE CONNECTOR'S OWN CONFIG KEY, NOT AN AUTOMATION. `rel.CONFIG_KEY` / + `rel.SYNC_PRESETS` are the same key and the same vocabulary `sync_seconds` reads and + `_store_resync_loop` sleeps on, so there is one value and the card reads back exactly what the + loop will use. Storing a cron on a synthetic automation would have been a second copy with a + guaranteed drift date. + + ⚠ EVERY OTHER FIELD IS REFUSED RATHER THAN IGNORED. A PATCH that silently kept only the part + it liked would let somebody rename this agent, watch the 200, and find the name gone on reload + — the failure shape this module has already paid for twice. + + ⚠ AND THE LATENCY IS REPORTED, because it is real and a person would otherwise call it a bug: + the loop re-reads the cadence at the TOP of its cycle and then sleeps, so a change takes effect + on the NEXT pass — up to one full OLD interval away (24 hours if it was on `daily`). + """ + import odoo_relational as rel # noqa: PLC0415 + + known = {"trigger", "every", "syncEvery"} + extra = sorted(k for k in body if k not in known) + if extra: + raise err(409, "system_agent", + "this agent is the Odoo connection's own sync. Only how often it runs can be " + f"changed here; {', '.join(extra)} belongs to the connector in Connectors") + every = str(body.get("syncEvery") or body.get("every") + or ((body.get("trigger") or {}) if isinstance(body.get("trigger"), dict) else {}) + .get("preset") or "").strip().lower() + if every not in rel.SYNC_PRESETS: + raise err(400, "bad_cadence", + f"pick one of: {', '.join(sorted(rel.SYNC_PRESETS))}") + + def _up(cur): + cur = dict(cur) if isinstance(cur, dict) else {} + cur["syncEvery"] = every + return cur + + session.runtime.update(rel.CONFIG_KEY, _up, flush="sync") + row = _odoo_sync_row(session, detail=True) + if row is None: + raise err(404, "unknown_automation", "no automation with that id") + secs = row["trigger"].get("everySeconds") + note = ("It syncs on a timer again from the next cycle." if secs + else "Nothing will sync on a timer now.") + return {"automation": row, + "notes": [f"The Odoo sync is set to {every}. {note} A change takes effect on the " + f"loop's next pass, so it can be up to one of the OLD intervals away."]} + + +def _triggers_vocab(session): + """C3's server-owned trigger list: `[{key, label, ready, needs, planned}]`, keys EXACTLY + `engine.TRIGGER_KEYS` + `engine.TRIGGER_PLANNED`. `ready:false` + `needs` renders as a + not-configured state — never a dead control, never a client-side union. + + ⭐ WAVE 23 (R2): the list is the WHOLE Airtable-parity vocabulary, and the two triggers we + have not built ride it with `planned: true`. That is the honest version of "show all, wire + eight": the picker paints them faded with a reason instead of a shorter list that quietly + implies the missing ones do not exist. `clean_trigger` refuses them, so the faded state is + enforced at the door and not merely in the client's `disabled` attribute. + """ + tick_on = _tick_state()["enabled"] + g = oauth_connect.status(session.runtime, session.uname).get("google") or {} + email_ready = bool(g.get("configured")) and bool(g.get("connected")) \ + and not g.get("reconnect") + email_needs = "" if email_ready else ( + "connect_gmail" if g.get("configured") else "configure_google") + per = { + "manual": (True, ""), + "schedule": (tick_on, "" if tick_on else "arm_tick"), + "event_field": (True, ""), + "record_updated": (True, ""), + "record_created": (True, ""), + "enters_view": (True, ""), + "webhook": (True, ""), + "email": (email_ready, email_needs), + "form_submitted": (True, ""), + # ⭐ WAVE 24 (C-TRIG) — Instagram discovery, now a trigger. Readiness is the vendor key, + # the same bit `paidReady` carries: with no key the search door is closed and the picker + # must say so rather than offering a control that silently finds nothing. `clean_trigger` + # still ACCEPTS it either way — readiness is a deployment fact, not a validity one, which + # is the same split `email` already makes. + "ig_profile_match": (engine.bd_ready(), "" if engine.bd_ready() else "configure_brightdata"), + # ⭐ WAVE 29 (D-9 / R1) — TikTok, live. ⛔ IT NEEDS ITS OWN ROW EVEN THOUGH THE ANSWER IS + # IDENTICAL, and the reason is the `.get(k, (True, ""))` default below: a trigger this dict + # forgets is reported READY, so a deployment with no vendor key would offer TikTok search + # as configured and the search would find nothing. Same key, same readiness bit, stated. + "tiktok_profile_match": (engine.bd_ready(), + "" if engine.bd_ready() else "configure_brightdata"), + } + #: ⭐ WAVE 24 — the server's own one-line description per trigger. C-TYPES: the picker renders + #: THIS under the option, because a CLIENT paraphrase of a server vocabulary is a second copy + #: of it, free to drift. Absent = the client shows nothing, never something invented. + detail = { + "manual": "It runs only when you press Run now", + "schedule": "It runs on a repeating schedule", + "event_field": "A record in the database starts matching a condition you set", + "record_updated": "Any of the columns you watch is changed", + "record_created": "A new record is added to the database", + "enters_view": "A record starts appearing in a saved view", + "webhook": "Something outside calls this automation's URL", + "email": "A message arrives in the connected mailbox", + "form_submitted": "Somebody submits one of this database's forms", + "ig_profile_match": "Search Instagram for profiles matching your filters, on a schedule", + "button_clicked": "Somebody presses a button on a record", + "comment_added": "Somebody comments on a record", + "web_page_changed": "A page you are watching is different from last time", + "tiktok_profile_match": "Search TikTok for profiles matching your filters, on a schedule", + } + + def _taxonomy(k): + """⭐ WAVE 25 · C2 — the four taxonomy keys, composed from the ENGINE's maps. + + ⛔ `.get(k) or FALLBACK`, NEVER `TRIGGER_GROUP_OF[k]`. The first draft of this indexed the + map on the reasoning that a default is how a Connector trigger quietly appears under + Database — and the gate rejected it, correctly, against the incident `per.get(k, ...)` + eight lines below records: a key added to `TRIGGER_KEYS` without remembering a dict beside + it raised KeyError and took `GET /automations` down, i.e. the whole surface, which polls + this every 2.5 s. A mis-grouped row is cosmetic; a 500 is not, and the ranking is not + close. + ⚠ THE CLASSIFICATION IS STILL MANDATORY — it is enforced at the GATE (no shipped trigger + may land in `other`) rather than at the request. Soft here, hard there. + """ + g = engine.TRIGGER_GROUP_OF.get(k) or engine.TRIGGER_GROUP_FALLBACK + return {"group": g, + "groupLabel": engine.TRIGGER_GROUPS[g]["label"], + "groupOrder": engine.TRIGGER_GROUPS[g]["order"], + # The SUB-group inside "Connector"; None everywhere else. ⚠ Group by this key, + # render its label — it is NOT a connector-directory slug (see the engine's note). + "connector": engine.TRIGGER_CONNECTOR.get(k), + # D-55: "the cron drives this one". Derived from the engine's schedule set MINUS + # `manual`, so the client's `CRON_DRIVEN_TRIGGERS` copy can be deleted. + "schedules": k in engine.TRIGGER_CRON_KEYS} + + out = [] + for k in engine.TRIGGER_KEYS: + # ⚠ `.get` WITH A DEFAULT, NOT `per[k]`. This loop walks the ENGINE's vocabulary and + # indexed a hand-maintained dict beside it: adding a key to `TRIGGER_KEYS` without + # remembering this dict raised KeyError and 500'd `GET /automations` — the payload the + # whole automation surface polls every 2.5 s — with every gate and `tsc` still green. + # Defaulting to "ready, needs nothing" is the honest fallback: a trigger the engine + # offers and this route has no readiness opinion about is simply available. + ready, needs = per.get(k, (True, "")) + row = {"key": k, "label": engine.TRIGGER_LABELS[k], "ready": ready, "needs": needs, + "planned": False, "detail": detail.get(k, ""), + # A3(3): the connect affordance is SERVER-COMPOSED — the client never maps a + # `needs` token to a route, so B's CONNECT_PROVIDERS shim deletes itself. + "connect": None, **_taxonomy(k)} + if not ready and needs == "connect_gmail": + row["connect"] = {"provider": "google", + "startUrl": "/api/v1/oauth/google/start"} + out.append(row) + for k in engine.TRIGGER_PLANNED: + out.append({"key": k, "label": engine.TRIGGER_LABELS[k], "ready": False, + "needs": "coming_soon", "planned": True, "connect": None, + "detail": detail.get(k, ""), **_taxonomy(k)}) + return out + + +def _tick_state(): + """⭐ WAVE 21 (C6 amendment A1) — can a SCHEDULE fire on this deployment? + + TWO independent paths can: the in-process scheduler (`AIOS_AUTOMATIONS=1`, + `automation_engine.py` module bottom) and an external cron POSTing `/automations/tick`, + gated on `AIOS_AUTOMATION_TICK_TOKEN` (AWS EventBridge in production). The Step-1 Trigger + card must be honest in both directions: "schedules won't fire" on a deployment where + EventBridge demonstrably fires them daily is the exact lie R9 forbids. `external` means + "the door is OPEN", never "the caller is alive" — the client's copy says so.""" + inproc = os.environ.get("AIOS_AUTOMATIONS") == "1" + ext = bool(os.environ.get("AIOS_AUTOMATION_TICK_TOKEN")) + return {"enabled": bool(inproc or ext), + "source": "in-process" if inproc else ("external" if ext else "")} + + +#: W29-T01 — the Board retirement ran, per tenant, this process. Same shape and same reasoning as +#: `_C8_MIGRATED` below: a one-way cleanup of data no living code path creates any more, whose +#: cost is a full walk of every row-cell in the tenant and whose result on pass 2..N is always +#: "nothing to do". ⚠ A module-global keyed by tenant is deliberately NOT reset by a table create +#: or delete — a new database cannot contain the legacy stage cells this retires. +_BOARD_RETIRED = set() + +#: ⭐ WAVE 35 · T37 — tenants whose statements agent has been considered THIS PROCESS. Same shape as +#: `_BOARD_RETIRED` and, like it, purely a cost saver: the DURABLE idempotency is the row's own +#: existence, which `engine.seed_statements_agent` re-checks inside its own store mutation. +#: ⚠ SO A RESTART RE-CHECKING IS HARMLESS BY CONSTRUCTION, which is the property to preserve. If +#: this set were ever the only guard, a process restart would mint a second agent. +_STATEMENTS_SEEDED = set() + +#: ⭐⭐ WAVE 30 · T12 — THE PICKER DEFAULT, MEMOISED. `{tenant: (stamp, table_key)}`, the shape +#: `scope_cache` stores. +#: +#: ⛔ WHY THE DERIVED STRING AND NOT THE DOCUMENT. The obvious cache here is the `user_tables` +#: bucket itself, and it is the wrong one: that bucket is up to 35.8 MB per tenant and this box is +#: the HF free tier, so caching it would trade a latency problem for a memory one. What the warm +#: path actually needs is `discover_default_table`'s ANSWER — one short string. +#: +#: ⛔ AND WHY NOT A PLAIN `_BOARD_RETIRED`-STYLE ONCE-PER-PROCESS SET, which would have been less +#: code: the election reads which profile databases exist and which hold rows, and BOTH change +#: while the process lives (somebody creates a database, an automation writes the first row). A +#: once-per-process memo would pin the picker's default to whatever was true at boot and never +#: correct itself — a default that disagrees with the save door, which is exactly the wave-25 R2 +#: defect the `"table"` line's own comment records. A TTL bounds the staleness instead. +#: +#: ⚠ `scope_cache` rather than a hand-rolled dict, because it is the house pattern for precisely +#: this (`routes_customers`, `routes_products`, `pages` all use it) and it is stale-while-refresh: +#: once a copy exists NO request blocks on a rebuild. Automation was the one module importing it +#: nowhere, which the wave-30 scout named as the reason every other surface feels fast. +_DISCOVER_DEFAULT = {} +#: 5 minutes — the same order as `apiBridge.ts:CUSTOMERS_FRESH_MS` on the client. Overridable so a +#: gate can pin it rather than sleep. +_DISCOVER_DEFAULT_TTL = float(os.environ.get("AIOS_AUTOMATION_DEFAULT_TTL") or 300) + + +#: The one store key `_LentTables` intercepts. DERIVED from the engine's own constant rather than +#: written out here: `engine.ut_all` reads the bucket through it, so if that key ever moves, the +#: lend moves with it instead of silently becoming a pass-through that still looks correct. +_UT_STORE_KEY = engine.UT_STORE_KEY + + +def _LentTables(runtime, tables): + """⭐⭐ WAVE 30 · T13, NOW W31-C1 — a read-only `st` that serves ONE already-read `user_tables` + document and passes every other key straight through to the real runtime. + + ⭐⭐ WAVE 31 · C1 — THE BODY IS NOW `core.user_tables.lend`, AND THE CLASS THAT USED TO BE HERE + IS GONE. W30's version carried its own note that the general fix was unavailable at the time: + *"adding a `tables=` parameter to `may_open` would mean editing platform/core/user_tables.py, + which belongs to another lane this wave."* It is session B's lane THIS wave, they built the + generalisation (`user_tables.lend` + the `_LENDABLE` allow-list) for W31-T10, and this is the + second caller adopting it rather than becoming a third copy of the shape. + + ⭐ AND THE SWAP FIXES SOMETHING THIS FUNCTION NEVER COVERED. The old class intercepted exactly + one key, so `may_open`'s LAST branch — `shares.may_see` → `role_for` → `st.get('object_shares')` + — still took a fresh read PER TABLE. `_LENDABLE` covers both buckets, so a shared database no + longer costs a second whole-document read per row of the picker. The name is kept because + `verify_automation`'s NC66 pins this symbol as the thing it replaces to restore the `1 + N` + state; keeping it a function means that control still has exactly one seam to swap. + + ⛔ THE OBVIOUS FIX IS STILL THE FORBIDDEN ONE. Inlining the creator-or-admin test here would + remove the N reads and re-create the exact defect wave 20 fixed: this route USED to carry its + own wider rule (`createdBy in (uname, 'automation', 'scheduler')`), so a non-admin saw a + database in the picker and was refused the moment they opened it. `may_open` stays THE one + resolver, unmodified and still called per table; it is simply no longer charged for a document + the caller is already holding. + """ + # ⚠ IMPORTED HERE, not at module scope — `core` is deliberately kept out of this module's + # import-time graph (the same reason `automation_tables` does it locally 300 lines down). + import core.user_tables as _ut + return _ut.lend(runtime, **{_UT_STORE_KEY: tables}) + + +def _discover_default(session, tables): + """The discovery picker's default table for this tenant — WITHOUT a bucket read on a warm call. + + ⚠ `tables` is the document the caller ALREADY holds on a cold call (the retirement pass reads + one). Passing it through means the cold path elects from the copy it has rather than taking a + second one, so this is never an extra read — only ever a saved one. + """ + if not session.runtime.available(): + return "" + if tables is not None: + # Cold call: the document is in hand. Elect from it and prime the memo in the same pass. + value = engine.discover_default_table(session.runtime, tables=tables) + _DISCOVER_DEFAULT[session.tenant] = (time.time(), value) + return value + return scope_cache.get(_DISCOVER_DEFAULT, session.tenant, _DISCOVER_DEFAULT_TTL, + lambda: engine.discover_default_table(session.runtime)) + + +def warm_default(rt): + """⭐⭐ WAVE 30 · T12, THE COLD HALF. Elect the discovery picker's default at BOOT. + + ⛔ THE HALF THE MEMO CANNOT FIX, and it is why this exists rather than being a nicety. The memo + above makes calls 2..N free; **call 1 is still a full `user_tables` download**, and it lands on + whoever clicks Automation first after a deploy — the one visitor with no cache anywhere, + waiting on a document whose documented ceiling is 35.8 MB / ~1.4 s, taken under the store's + single lock. That is the owner's *"only automation has a loading screen"* on a cold Space, and + two waves of memoising the warm path could never touch it. Automation was the only module + importing `scope_cache` nowhere AND the only one absent from `_prewarm`. + + ⭐ IT ELECTS, IT DOES NOT CACHE THE DOCUMENT — deliberately, and this is the whole design. + `discover_default_table` reads the bucket once here, in the prewarm daemon thread where nobody + is waiting, and what survives is a DERIVED STRING. Holding the 35.8 MB document resident would + trade a latency the tenant notices for memory the HF free tier does not have, which is the + argument `_DISCOVER_DEFAULT`'s own note makes against caching it. + + ⚠ SAME TTL AS THE REQUEST PATH, not a permanent set: the election reads which databases exist + and which hold rows, and both change while the process lives. Priming by assignment is exactly + what the cold request path already does one function up, so there is one way this memo is + filled, not two. + + Returns the elected key (`""` when the store is unavailable), so a caller can log it rather + than guess whether the warm-up did anything. Called from `main.py:_prewarm` only. + """ + tenant = str(getattr(rt, "key", "") or "") + if not tenant or not rt.available(): + return "" + value = engine.discover_default_table(rt) + _DISCOVER_DEFAULT[tenant] = (time.time(), value) + return value + + +@router.get("/automations") +def list_automations(session: Session = Depends(_GATE)): + """`{automations: [...], kinds: [...], cronPresets: [...]}` — the rail's whole payload. + + The vocabularies ride WITH the list rather than sitting in a client constant: the cron + presets and the kind list are the server's, and a client copy of either is a thing that goes + stale silently (the editor would offer a preset the parser rejects).""" + # ⭐⭐ WAVE 29 (W29-T01, owner item 5: "Automation takes a while to appear"). THE COMPLAINT WAS + # THIS ROUTE, and the cost was never the automations: it read the whole `user_tables` bucket + # THREE TIMES per call — once for the stage-field retirement scan, twice more inside + # `discover_default_table`. That bucket's documented ceiling is 35.8 MB / ~1.4 s to serialize + # (`automation_engine.py` header) and `core/store.py` re-serializes on EVERY `.get()`, hit or + # miss, UNDER THE STORE LOCK — so the reads also serialize behind each other. ~4 s of deep + # copying to render a rail that shows a name and a toggle. + # + # Now: ONE read, lent to both callers. And the retirement SCAN — O(all row-cells in the + # tenant), which hits its own `if not stage_keys: return` only AFTER walking every row of + # every table — runs once per tenant per process, mirroring the `_C8_MIGRATED` guard below. + # + # ⛔ WHY SKIPPING THE SCAN ON CALLS 2..N IS SAFE, and it is not "because it is idempotent": + # `all_definitions` STRIPS retired board state on every read (`automation_engine.py`'s + # `_without_retired_board`), so the persist step is hygiene, not correctness. No client can + # ever be shown state this skip left behind. Nothing writes new `stage_auto_` cells either — + # wave 26's R6 deleted the board that made them. + # ⭐⭐ WAVE 30 · T12 (owner items 4/5, the complaint that has now survived TWO waves). + # ⛔ W29-T01 GUARDED THE SCAN AND NOT THE READ, and that is the whole of what was left. The + # line below used to be unconditional — every single request deep-copied the tenant's entire + # `user_tables` document (documented ceiling 35.8 MB / ~1.4 s, and `core/store.py:Store.get` + # re-serializes on EVERY `.get()`, hit or miss, UNDER THE STORE LOCK so the copies also queue + # behind each other) — while on calls 2..N the result was DISCARDED: `tables` had exactly two + # consumers, the `_BOARD_RETIRED` scan (skipped after call 1) and a picker DEFAULT STRING. + # A whole-tenant document, per request, to render a rail showing a name and a toggle. + tables = None + if session.runtime.available() and session.tenant not in _BOARD_RETIRED: + tables = engine.ut_all(session.runtime) + _BOARD_RETIRED.add(session.tenant) + # Idempotent Board retirement removes only engine-marked stage fields and legacy Board + # state. User-created Status/Stage columns remain intact. + engine.retire_automation_board_state(session.runtime, tables=tables) + # ⭐⭐ WAVE 35 · T37 / OWNER RULING R10 — ROYAL IMPORTS' STATEMENTS AGENT EXISTS BEFORE ANYBODY + # ASKS FOR IT. It is minted ONCE, by the container, SWITCHED OFF; every subsequent call finds + # it present and returns immediately. See `engine.seed_statements_agent` for why this one is + # STORED where wave 34's Odoo agent is derived, and why the row's existence is a sufficient + # idempotency key (it cannot be deleted). + # ⚠ GUARDED SO IT CAN NEVER TAKE THE RAIL DOWN. A tenant with no Odoo, a store mid-outage or a + # validator change must degrade to "no statements agent", never to a 500 on the one route the + # whole Agents surface polls. The same posture `_odoo_sync_row` takes for the same reason. + if session.tenant not in _STATEMENTS_SEEDED: + try: + engine.seed_statements_agent(session.runtime, username=session.uname or "system") + except Exception: # noqa: BLE001 + pass + _STATEMENTS_SEEDED.add(session.tenant) + defs = engine.all_definitions(session.runtime) + items = [_wire(d, session.tenant, rt=session.runtime) for _, d in + sorted(defs.items(), key=lambda kv: (kv[1].get("name") or "").lower())] + # ⭐⭐ WAVE 34 · CONTRACT C3 (W34-T48) — field agents join the list as SYNTHETIC rows. + # ⚠ MERGED AND RE-SORTED, not appended in a block at the end. R13 asks for a field agent to be + # "VIEWABLE under the Agent module", i.e. one list of agents, not a list with a second list + # stapled to it — and a rail that sorts by name everywhere except its last few rows reads as a + # rendering bug. `_field_agent_rows` costs no whole-document read; see its docstring. + _sys_rows = [r for r in (_odoo_sync_row(session),) if r] + items = sorted(items + _field_agent_rows(session) + _sys_rows, + key=lambda r: str(r.get("name") or "").lower()) + return {"automations": items, + # ⭐ WAVE 24 — DERIVED from the engine's own `KINDS`, not a hand-written trio. It was + # three literals that happened to match, which is a second copy of a server + # vocabulary; R6 has just made two of them uncreatable and `plain` has joined, so a + # hand list would now be wrong in three ways at once. `creatable` carries R6 onto the + # wire, so the ruling is a fact the client can read rather than one it must remember. + "kinds": [{"key": k, "label": engine.KIND_LABELS.get(k, k), + "creatable": k not in engine.RETIRED_KINDS} + for k in engine.KINDS], + "cronPresets": engine.CRON_PRESETS, + # ⚠ A BOOLEAN, NEVER THE KEY. The surface needs to say "the paid rung is not + # configured" honestly instead of offering a tier that will silently refuse — and + # that needs exactly one bit. Shipping the key itself to a browser would put a + # billable secret in every user's devtools. + "paidReady": engine.bd_ready(), + # THE SOURCE REGISTRY (D-9's seam), on the wire for the same reason `cronPresets` is: + # the module that RUNS a source is the only thing entitled to say what it can do, and + # a client copy of "Instagram can discover, TikTok cannot" goes stale in silence. + "sources": engine.source_status(), + # The discovery vocabulary, likewise server-owned: every name here is MEASURED- + # accepted by the vendor's own validator, so a client that invented one would build a + # query the API rejects. `lead` is the subset seen carrying VALUES on real rows. + # ⭐⭐ WAVE 32 · T46 (D-167) — THE VOCABULARY HAS A PLATFORM, AND `byKind` IS ADDITIVE + # ON PURPOSE. `fields`/`lead` keep INSTAGRAM's 21 and 3, so no stored automation and no + # client that has not adopted this changes behaviour today; `byKind` carries the per- + # corpus answer, and the SERVER already refuses a TikTok predicate naming one of the 16 + # fields TikTok's dataset does not have (`clean_predicates(..., kind)`). The door is + # closed either way — this is what lets the Find panel stop OFFERING them. + # ⚠ Derived through `engine.filter_fields`, the same accessor the validator uses, so + # the published vocabulary and the enforced one cannot drift — which is precisely what + # D-167 was: a route serving 21 names and a validator checking the same 21, both wrong + # about TikTok together, with nothing able to notice. + "discoverByKind": {k: {"fields": list(engine.filter_fields(k)[0]), + "lead": list(engine.filter_fields(k)[1])} + for k in engine.DISCOVERY_KINDS}, + "discover": {"fields": list(engine.BD_FILTER_FIELDS), + "lead": list(engine.BD_FILTER_LEAD), + "operators": list(engine.BD_FILTER_OPS), + "nullaryOperators": list(engine.BD_NULLARY_OPS), + "maxRecords": engine.BD_MAX_RECORDS, + # Wave 22 C4 — ADDITIVE: per-field flags for the toggle rows (the 3 + # PII fields stay structurally absent, R3) + the too-big guard's + # numbers, so the surface can say WHY a filter is refused before the + # server has to. + "filterMeta": engine.filter_meta(), + "guard": {"minNarrowing": engine.BD_MIN_NARROWING, + "maxRecords": engine.BD_MAX_RECORDS}, + # ⛔ D-59's `categoryOptions` IS DELIBERATELY NOT HERE — it is served by + # `GET /automations/discover/categories`. It sat on this payload for one + # commit and that was a real defect: THIS ROUTE IS POLLED EVERY 2.5 s by + # the whole automation surface, and deriving the options reads two user + # tables plus the PLATFORM MASTER, which is a different HF repo — i.e. a + # network round-trip per poll, per open tab. Caught by the gate's own + # output, which started carrying `store:get:master_snapshots` errors from + # a suite whose stated contract is that it touches no network. + # C6/R5's vocabulary, so the seed picker cannot offer a source the + # validator refuses. + "seedSources": list(engine.SEED_SOURCES), + "seedMaxRows": engine.SEED_MAX_ROWS, + # W29-T01: `tables` is the snapshot read once at the top of this handler. + # ⚠ IT WAS TAKEN BEFORE THE RETIREMENT WROTE, and that is deliberate and + # harmless: retirement only removes `stage_auto_*` FIELDS and pops those + # same keys off rows, and the election reads the preset-profile vocabulary + # (`handle` + N preset keys) and whether a table has ANY rows. A stage key + # is in neither set, and no row is ever deleted — so the pre-write + # snapshot and the post-write bucket cannot elect different tables. + # 2026-08-10 — the OFFER must name the table the SAVE will actually use. + # This was the bare `DISCOVER_TABLE` constant while `create`/`patch` now + # resolve a targetless discovery flow to the profile database the tenant + # already has, so the picker would have shown `ut_ig_candidates` and the + # save would have written somewhere else — a default that disagrees with + # itself across two panels, which is the shape wave 25's R2 fixed for + # `targetTable` vs the action's `table`. + # ⭐ WAVE 30 · T12 — through the per-tenant memo. The ELECTION rule and + # everything the comment above says about it are unchanged; what changed + # is that a warm request no longer re-reads a 35.8 MB document to + # recompute a string that did not move. + "table": (_discover_default(session, tables) + or engine.DISCOVER_TABLE)}, + "storeAvailable": bool(session.runtime.available()), + # Wave 22 C3 — the trigger vocabulary, session-scoped because email readiness is a + # per-USER fact (the poll runs through the creator's own Gmail connection). + "triggers": _triggers_vocab(session), + # ⭐ WAVE 23 C4 (R3) — the ACTION MENU, including what we have not built. Each row + # carries `ready`, so B paints "Send email" and "Run script" faded with the server's + # own reason instead of omitting them — the owner asked for Airtable's full menu, and + # a shorter list would imply those actions do not exist. `clean_actions` REFUSES an + # unready kind, so the faded state is a wall rather than a styling choice. + # ⭐ W35-T35 (C8): `session.runtime` is what withholds a TENANT-GATED row. Passing it + # here is the whole of "the menu does not offer Send statements to another tenant"; + # the STORE-side wall is `clean_actions(rt=)`, deliberately separate, because a picker + # is not a security boundary [[opening-a-route-widens-every-field]]. + "actionsCatalog": engine.action_catalog(session.runtime), + # The builder's own vocabulary: how deep a condition tree may nest, how deep groups + # may nest, and the ceilings. B reads these instead of hard-coding the same numbers + # into its "+ Add condition" affordance. + "flow": {"condOps": list(engine.LANE_OPS), + "nullaryCondOps": list(engine.LANE_NULLARY_OPS), + "maxCondDepth": engine.MAX_COND_DEPTH, + "maxCondChildren": engine.MAX_COND_CHILDREN, + "maxGroupDepth": engine.MAX_GROUP_DEPTH, + "maxActions": engine.MAX_ACTIONS, + # ⭐ W29-T09 (owner item 11) — THE POST-GROUP VOCABULARY, on the wire for the + # same reason `cronPresets` is: `clean_post_groups` REFUSES a type it does + # not know, so a client that invented one would build a config the save door + # rejects with a sentence about a word the person never typed. Both halves + # ride — the stored key and the label a person reads — because a client-side + # translation of `video` into "Reels" is a second copy of this list. + "postTypes": [{"key": t, "label": engine.POST_TYPE_LABELS.get(t, t)} + for t in engine.POST_TYPES], + # The ceiling a group's limit is judged against. `clean_post_groups` bounds a + # group by the action's OWN `maxPosts`, not by a constant, so the control can + # only warn honestly if it reads the same number the validator uses. + "maxPostsPerPull": engine.MAX_POSTS_PER_PULL, + # ⭐⭐ W33-T52 (owner item 16) — WHICH CONFIG KEYS A KIND REQUIRES, on the + # wire, for exactly the reason `postTypes` above is: the panel paints a red + # `*` beside a required control, and a client-side table of which keys those + # are would be a SECOND copy of `ACTION_REQUIRED` living in another file. The + # two would agree on the day they were written and diverge the first time a + # kind gained a key — the panel would then mark a control optional that the + # runner blocks on, and the person would read "this is fine" from the one + # surface that is meant to tell them it is not. + # ⛔ BOTH HALVES RIDE, phrase AND key, because the phrase is the ONLY human + # wording of that requirement anywhere ("a column to write into"), and the + # runner's own refusal sentence is built from it. A client that re-worded it + # would give the same requirement two names. + # ⚠ It is `ACTION_REQUIRED`, not `WEB_REQUIRED`: the table is the one the + # run refusal and `unconfigured_actions` already read, so a kind added to it + # later paints its `*` with no client change at all. + "actionRequired": {kind: [{"phrase": phrase, "key": key} + for phrase, key in reqs] + for kind, reqs in engine.ACTION_REQUIRED.items()}}, + # Wave 21 C6-A1: {"enabled": bool, "source": "in-process"|"external"|""} — see + # `_tick_state` for why one boolean off AIOS_AUTOMATIONS alone would lie. + "tick": _tick_state()} + + +@router.post("/automations/discover/estimate") +def discover_estimate(body: dict = Body(default=None), session: Session = Depends(_GATE)): + """What would this search cost? Shown BEFORE the run, never after. + + ⚠ THE ANSWER IS AN ESTIMATE AND SAYS SO IN ITS OWN PAYLOAD (`basis: "SPEC"`). Bright Data + never returns a price before a run — the funds gate fires first and `price: 0` means "not + priced", not "free" — and this account's token cannot read a balance (`/customer/balance` + answers 403). A number presented as billed would be the invented measurement this whole + module refuses to make. + """ + return engine.discover_estimate((body or {}).get("recordsLimit")) + + +#: C8's migration ran, per tenant, this process. Once is the point: the sweep only writes when +#: an unbound bag exists, so after the first pass this is a read that finds nothing. +_C8_MIGRATED = set() + + +def _table_views(session, key): + """⭐ WAVE 24 (item 8) — the saved views on ONE user table as `[{id, label}]`, or **None** + when the workspace bucket could not be read at all. + + The client said "This server did not offer a view list" because the payload genuinely had + no `views` key. C-TYPES: **absent stays ABSENT.** `[]` is a MEASUREMENT ("this database has + no saved views") and `None` is a STATE ("nobody looked") — collapsing them is precisely how + an empty picker comes to read as a claim about the database. + + ⛔ SOURCED FROM `core.table_store`, WHICH IS THE READER `view_filter` ALREADY USES — not a + second one, the same one, listed instead of looked up. The brief said to source it the way + the grid does; the grid's projection (`aios_grid.views_from_defs`) is the wrong list HERE and + the difference matters: it INJECTS the system view ("All records") and PROJECTS cohorts as + views, and neither of those lives in the stored bucket — so `view_filter` answers "view no + longer exists" for every one of them. A picker built from that projection would offer options + the `enters_view` trigger cannot resolve, which is a worse bug than the missing key it fixes. + (An `enters_view` trigger on "All records" would also mean "fire on every record", so its + absence is correct rather than a gap.) + + ⚠ `consume_corrections=False`: this is a READ for a picker, and the default MUTATES — it + takes and clears the pending field-correction acknowledgement, so listing views would eat a + protocol message meant for the grid. + """ + try: + import core.table_store as table_store + # ⛔ THE REACHABILITY PROBE IS NOT REDUNDANT, and leaving it out was a real defect this + # gate's own NC caught: EVERY `TableStore` accessor wraps its store read in `try/except` + # and returns `{}` on failure. So a bucket that could not be read is indistinguishable + # from one with no views — which collapses exactly the ABSENT/EMPTY distinction this + # function exists to preserve, and the caller would ship `views: []` as a measurement + # nobody took. The one read that is allowed to raise has to be ours. + session.runtime.get(f"{key}_table_workspace") + tops = table_store.make(f"{key}_table_workspace", st=session.runtime) + own = (tops.workspace(session.uname, consume_corrections=False) or {}).get("views") or {} + shared = tops.shared_views(session.uname, session.admin) or {} + except Exception: # noqa: BLE001 + return None + merged = {**shared, **own} # a view lives in ONE home; the merge is belt-and-braces + return sorted( + ({"id": str(vid), "label": str((v or {}).get("name") or vid)} + for vid, v in merged.items() if isinstance(v, dict)), + key=lambda r: r["label"].lower()) + + +@router.get("/automations/tables") +def automation_tables(session: Session = Depends(_GATE)): + """The blank databases an automation can target, with their fields. + + ⚠ A READ-ONLY MIRROR of C3-UT's `GET /api/v1/tables`, under this router's own prefix so the + two can never collide. It exists because the automation editor needs the table+field list to + build a config at all, and D must not block on A's route landing. When C3-UT is live this + keeps working (same bucket) — it is a duplicate reader, never a second writer. + + ⛔ WAVE 20, item 3 — IT NO LONGER MIRRORS THE WALL, IT CALLS IT. This route re-implemented + `may_open` and got it WIDER: it also admitted `createdBy in ('automation', 'scheduler')`, so + a non-admin saw automation-created databases here and was refused the moment they opened, + edited or deleted one. A duplicate READER is fine; a duplicate WALL is not, because the two + only disagree in front of a user. One resolver now, and the engine stamps a human owner so + the merge takes nothing legitimate away (`ut_ensure`, `MACHINE_OWNERS`). + """ + import core.user_tables as user_tables + + # C8 (wave 22): bind pre-law automation bags to the definitions that write them — once + # per tenant per process, and a no-op read after the first real pass. A write-on-read, + # stated out loud: this is the surface whose stale bags mislead (the column picker), so + # it is where the truth gets repaired. + if session.tenant not in _C8_MIGRATED and session.runtime.available(): + _C8_MIGRATED.add(session.tenant) + try: + engine.bind_unbound_fields(session.runtime) + except Exception: # noqa: BLE001 + pass + + out = [] + # ⭐⭐ WAVE 30 · T13 — ONE read of the tenant document, LENT to the wall for every table. + # It was `1 + N` full deep copies (this `ut_all`, then `may_open` → `get` → `all_tables` per + # table), each of a document with a 35.8 MB / ~370 ms ceiling, all of them queued behind + # `Store._lock`. `may_open` is unchanged and still asked about every table — see `_LentTables` + # on why re-implementing the wall here is the one fix that is NOT available. + _tables_doc = engine.ut_all(session.runtime) + _lent = _LentTables(session.runtime, _tables_doc) + for key, t in sorted(_tables_doc.items(), + key=lambda kv: (kv[1].get("label") or "").lower()): + if not user_tables.may_open(key, session.uname, session.admin, st=_lent): + continue + row = {"key": key, "label": t.get("label") or key, + "source": t.get("source") or "Blank", + "rowCount": len(t.get("rows") or {}), + "fields": [{"key": f.get("key"), "label": f.get("label"), + "type": f.get("type") or "text", + "automation": f.get("automation") or None} + for f in (t.get("fields") or [])]} + views = _table_views(session, key) + if views is not None: + row["views"] = views # absent when the bucket did not answer — never [] + out.append(row) + return {"tables": out} + + +@router.get("/automations/presets") +def automation_presets(table: str = "", session: Session = Depends(_GATE)): + """⭐ WAVE 25 · C1 / R2b — the Instagram preset columns, diffed against ONE database. + + `{fields: [{key, label, type, present}], willUse: [...], willCreate: [...]}` — the + "already in this database" / "will be created" split the owner asked for, so the Create record + action's configuration can SHOW what pointing it here does before anything is spent. + + ⛔ DECLARED ABOVE `/automations/{auto_id}`, AND THAT IS LOAD-BEARING RATHER THAN TIDY. + FastAPI matches routes in DECLARATION order, so a literal path registered after a sibling + path-parameter route is never reached — this handler would simply never run and + `get_automation` would answer "no automation with that id" for the id `presets`. A 404 with a + plausible sentence is the worst possible failure here, because it reads as "the endpoint is + fine, the data is missing". `/automations/tables` sits above the same route for the same + reason; this follows it rather than inventing a second convention. + + ⚠ WALLED BY `may_open`, LIKE EVERY OTHER TABLE READER. Without it, asking about a table you + cannot open would answer which of its columns exist — a small disclosure, and exactly the + duplicate-wall mistake `automation_tables` records at wave 20. + """ + import core.user_tables as user_tables + + key = str(table or "").strip() + if key and not user_tables.may_open(key, session.uname, session.admin, + st=session.runtime): + raise err(404, "unknown_table", "no such database") + return engine.preset_plan(session.runtime, key) + + +@router.get("/automations/discover/categories") +def automation_categories(session: Session = Depends(_GATE)): + """⭐ DEBT D-59 — the Category combobox's options: the values this deployment has ACTUALLY + SEEN, each with its observed count. `{options: [{value, count}]}`. + + ⛔ ITS OWN ROUTE, ON PURPOSE. This belongs to the Find surface and is asked for when that + panel opens — it must never ride `GET /automations`, which the whole automation surface polls + every 2.5 s: deriving these reads two user tables AND the platform master (a different HF + repo), so on the polled payload it is a network round-trip per tab per poll. + + ⚠ THE CONTROL STAYS A COMBOBOX. These are the values we have seen, not the values that exist. + D-59 is explicit that transcribing Instagram's published taxonomy would be worse than no + dropdown — a filter on a value the corpus does not use returns zero rows and looks exactly + like an honest "no such accounts exist", which misleads precisely when it looks authoritative. + """ + return {"options": engine.observed_categories(session.runtime)} + + +@router.post("/automations/seed/derive") +def automation_seed_derive(body: dict = Body(default=None), session: Session = Depends(_GATE)): + """⭐ WAVE 25 · C6 / R5 — "find me more accounts like the ones in this view". + + Body `{source: "view"|"cohort", table: "", id: ""}` → + `{derived: [...], basis: {rows, fields:[{name,label,value,coverage}], related, note}}` + + ⛔ IT DERIVES AND RETURNS; IT SAVES NOTHING. R5: the derived conditions are "visible and + editable, never hidden" — so the client drops them into the ordinary condition rows, where the + user edits them like anything else, and the ordinary PATCH stores them. A door that both + derived and saved would make the suggestion feel like a decision. + + ⚠ `basis` IS NOT DECORATION AND MUST BE RENDERED. It carries how many rows were read and the + MEASURED coverage of each characteristic, which is the difference between "12 of 12 of these + bios say florist" and "7 of 12 do" — presented identically, the weak one reads as authority. + + ⚠ Declared ABOVE `/automations/{auto_id}` for the reason `/automations/presets` is (FastAPI + matches in declaration order); that ordering is gated. + """ + import core.user_tables as user_tables + + body = body or {} + table = str(body.get("table") or "").strip() + if not table: + raise err(400, "no_table", "name the database to read the seed records from") + if not user_tables.may_open(table, session.uname, session.admin, st=session.runtime): + raise err(404, "unknown_table", "no such database") + source = str(body.get("source") or "view").strip().lower() + if source not in engine.SEED_SOURCES: + raise err(400, "bad_seed_source", + f"a seed comes from one of: {', '.join(engine.SEED_SOURCES)}") + rows, problem = engine.seed_rows(session.runtime, table, str(body.get("id") or "")) + if problem: + raise err(400, "bad_seed", problem) + derived, basis = engine.seed_predicates(rows) + return {"source": source, "table": table, "id": str(body.get("id") or ""), + "derived": derived, "basis": basis} + + +#: The injected chat transport, for gates only — `None` in every shipped path, so the route takes +#: the real ladder. Set by `verify_automation.py` to prove this door end to end with NO API key and +#: NO spend (`routes_query._call_model`'s `chat` argument is the same idea and the same reason). +_DRAFT_CHAT = [None] + + +@router.post("/automations/draft") +def draft_automation(body: dict = Body(default=None), session: Session = Depends(_GATE)): + """⭐⭐ WAVE 33 · W33-T54/T55 (owner item 7, ruling R3) — A DESCRIPTION BECOMES A DRAFT FLOW. + + `{prompt}` → `{draft: {name, trigger, table, actions:[{id,kind,config,why}]}, provider, + dropped: [...], saved: false}` — or a 400 carrying ONE plain sentence. + + ⛔⛔ NOTHING IS SAVED HERE, AND THE `saved: false` ON THE WIRE IS NOT DECORATION. T54's + `done-when` is *"gets back a DRAFT flow they can see BEFORE anything is saved"*; a door that + wrote first and showed second would satisfy every check about the flow's contents and none about + the promise. Accepting a draft is the ORDINARY `POST /automations` — R3's *"indistinguishable + from a hand-built one"* is achieved by there being no second write path, not by making the + second one look similar. + + ⛔⛔ CONTRACT C7, AND THIS ROUTE IS WHERE IT IS ENFORCED: *"the output must pass `clean_actions` + unchanged."* So it is RUN, here, before the person ever sees the draft — and the result is + DIFFED against what the model wrote. `clean_actions` has no disclosure channel (D-75): it drops + a config key it does not recognise and answers 200. Without this diff a person would accept a + flow, open it, and find a step configured differently from the one they were shown, with nothing + anywhere having said so. `dropped` is that channel, built here because this is the first caller + that needed one. + ⚠ A draft whose actions `clean_actions` REFUSES outright is a 400, never a partial flow: half a + flow presented as a whole one is the one outcome worse than a refusal. + + ⚠ DECLARED ABOVE `/automations/{auto_id}`, and that is load-bearing rather than tidy — FastAPI + matches in DECLARATION order, so a literal path registered after a sibling path-parameter route + is never reached, and `get_automation` would answer *"no automation with that id"* for the id + `draft`. The same reason `/automations/tables` and `/automations/presets` sit up here. + """ + prompt = str((body or {}).get("prompt") or "").strip() + if not prompt: + raise err(400, "no_prompt", "type what you want the automation to do") + # ⭐⭐ ASK D-18 (2026-08-18) — THE KEY THE CLIENT SENDS IS NOW READ. This door took `prompt` off + # the body and nothing else, so the Agent chat's model toggle (W36-T34) was a control that + # configured nothing: the value was accepted and dropped, which is the shape three tickets in + # this wave already tripped over. D held T34 rather than mark a picker BUILT while its value + # stopped at the browser, and was right to. + # ⚠ UNKNOWN OR UNCONFIGURED FALLS BACK TO THE LADDER, never refuses (D-18's asked-for posture, + # and C5's everywhere else): a model the ladder stops offering must not turn every later draft + # into an error. The response already names the rung that ANSWERED, which is the honest half. + _model = str((body or {}).get("model") or "").strip().lower() + if _model in ("", "auto"): + _model = None + # ⚠ THE TENANT'S OWN TABLES, THROUGH THE EXISTING WALL. `automation_tables` applies `may_open` + # per table, so the model is shown exactly the databases this caller may already see and cannot + # name one they were not granted — the permission wall re-used, never a second one built beside + # it. (`QueryPage`'s `granted` prop carries the same clause on the client side.) + tables = (automation_tables(session) or {}).get("tables") or [] + # ⭐⭐ WAVE 34 · W34-T42 / R18 — THE DRAFTER IS OFFERED WHAT THE PICKER OFFERS, nothing more. + # + # R18 collapsed six web actions into one on the menu. The other five stay in the catalog + # because the surviving "Web agent" composes its own steps out of them server-side — but a + # DRAFT is read and accepted by a PERSON, who then edits it on the Canvas. Handing the model + # the unfiltered catalog would let it draft a step kind that is not in the action picker: the + # reader could not add a second one, could not reason about where it came from, and the flow + # they accepted would not be one they could have built by hand. R3's *"indistinguishable from + # a hand-built one"* is a claim about what a person can REACH, not only about the write path. + # ⚠ This is the ONE place the filter is applied on this route, and it is applied to the + # drafter's vocabulary only. `clean_actions` below still validates against the FULL catalog, + # so a flow that already holds a hidden kind is unaffected and nothing 400s (D-65). + # ⚠ AND `required` IS NARROWED WITH IT, which is the half that is easy to miss. `draft_flow` + # builds the ENUM from `catalog` and the "settings you must write" prose from `required`, so + # filtering one and not the other describes settings for kinds the model cannot choose. That is + # not merely untidy: it spends prompt on unreachable options and invites the model to reach for + # one. `_ai_agent_plan` already narrows both in exactly this way; this follows it. + # W35-T35 (C8): tenant-gated rows are withheld here too, or the DRAFTER would offer a kind + # the same tenant's own save door then refuses — a flow the assistant proposes and the product + # will not accept. + menu_catalog = [r for r in engine.action_catalog(session.runtime) if r.get("menu") is not False] + _offered = {r["kind"] for r in menu_catalog} + draft, refusal, provider = ai_review.draft_flow( + prompt=prompt, + catalog=menu_catalog, + required={k: v for k, v in engine.ACTION_REQUIRED.items() if k in _offered}, + triggers=_triggers_vocab(session), + tables=tables, + # ⭐ W35 · CONTRACT C7 (`NOTE E-16`) — the spend is ATTRIBUTED. Unlike the two engine call + # sites, this one has a real person behind it: somebody typed the sentence, so `user` is + # the caller rather than the automation's owner. + st=session.runtime, user=getattr(session, "uname", "") or "", + model=_model, + chat=_DRAFT_CHAT[0]) + if refusal or not draft: + raise err(400, "draft_refused", refusal or "no automation could be drafted from that") + + # ── C7: run the real save-door validator and DIFF it ────────────────────────────────── + wrote = draft.get("actions") or [] + # W35-T35 (C8): the SAME `rt` the menu was built with. Without it this validator would refuse + # a tenant-gated kind it had just offered the model — the draft door disagreeing with the save + # door about one tenant, which reads as "the assistant produced an invalid flow". + cleaned, why = engine.clean_actions([{"kind": a.get("kind"), "config": a.get("config") or {}} + for a in wrote], rt=session.runtime) + if why or cleaned is None: + raise err(400, "draft_invalid", + f"the assistant produced a flow this deployment will not accept: {why}") + dropped = [] + for before, after in zip(wrote, cleaned): + b_cfg = before.get("config") or {} + a_cfg = after.get("config") or {} + gone = sorted(k for k in b_cfg + if k not in a_cfg or str(a_cfg.get(k)) != str(b_cfg.get(k))) + if gone: + dropped.append({"kind": str(after.get("kind") or ""), "keys": gone}) + # ⛔⛔ WHAT IS STILL MISSING FROM EACH STEP, PER STEP — and this is the fix for the sharpest + # thing the verifier found. The system prompt TELLS the model *"leave one blank rather than + # inventing a web address, a CSS selector or a column name"* (which is right), and the panel + # skips empty values (which is also right, on its own). Together they meant a `web_read` with + # no selector and no target column painted as a FINISHED step — url, attr, timeout, nothing + # else — and the ordinary create door then accepted it. A person approved a step that reads + # nothing into nowhere, having been shown no blank at all. + # ⚠ `engine.action_needs` IS THE PREDICATE, not a second list: the same function the builder's + # Configured/Unconfigured label and the run refusal read. Three readers, one answer. + _tables_by_key = {str(t.get("key")): t for t in tables} + _target_cols = {str(f.get("key")) for f + in (_tables_by_key.get(str(draft.get("table") or "")) or {}).get("fields") or []} + shown = [] + for i, after in enumerate(cleaned): + row = dict(after, why=str((wrote[i] or {}).get("why") or ""), + needs=engine.action_needs(after)) + # ⛔ AND A COLUMN NAME THE DATABASE DOES NOT HAVE. `field` is a free string that nobody + # validated at draft, at accept or at store — so a model writing the column's LABEL + # ("Price") instead of its key ("price"), which is exactly what a person would say and an + # obedient model would echo, stored a write into a column that does not exist. No schema + # was violated; the flow was well-formed and did something other than what was asked. + _f = str((after.get("config") or {}).get("field") or "") + if _f and _target_cols and _f not in _target_cols: + row["unknownField"] = _f + shown.append(row) + # ⚠ A TRIGGER THE CREATE DOOR WOULD REFUSE IS CORRECTED HERE, NOT SHOWN AND THEN 400'd. The + # enum makes this need a model that ignores its own schema, but the failure mode is ugly: the + # draft paints, the person clicks Accept, and the save refuses with a sentence about a word + # they never chose. Falling back to `manual` and SAYING SO keeps the draft usable. + _trig = str(draft.get("trigger") or "").strip() + _notes = [] + if _trig and _trig not in tuple(engine.TRIGGER_KEYS): + _notes.append(f"the assistant asked for a trigger this deployment does not have " + f"({_trig}). Set to manual instead") + _trig = "" + _asked = int(draft.get("asked") or len(wrote)) + if _asked > len(shown): + _notes.append(f"the assistant wrote {_asked} steps and a draft carries at most " + f"{ai_review.MAX_DRAFT_ACTIONS}; the last {_asked - len(shown)} were not " + f"kept. Describe the job in two automations, or shorten it.") + # ⭐⭐ WAVE 34 · W34-T47 (mailbox D-5) — THE MODEL'S OWN WORDS ARE NORMALISED, BECAUSE R6 + # CANNOT BE ENFORCED BY A SOURCE SCAN. + # + # ⛔ `web_prose` READS FILES. Every string below is written by a language model at run time, + # so the gate is structurally blind to it: the sweep every lane did this wave is undone by our + # own drafter the first time it answers with an em dash. Lane D MEASURED that a prompt + # instruction does not hold (their system prompt says "Never use an em dash" and the very next + # cerebras turn came back with one), so this is code, at the boundary, not a nicer prompt. + # ⚠ THREE STRINGS REACH A SCREEN FROM HERE and all three are covered: the flow's NAME, each + # step's `why`, and the `dropped`/`notes` sentences. `W34-T42` is what made them visible. + # ⚠ A DIGIT RANGE IS A DIFFERENT SENTENCE: "10-20" means "10 to 20", and rewriting it as + # "10, 20" states two numbers where the model stated a span. It gets its own rule, first. + # Same shape as `routes_query._no_dashes`, deliberately: one behaviour, two doors. + # ⚠ MODULE-LEVEL, not a closure. A normaliser defined inside this handler is unreachable to + # anything that is not an HTTP request, so the only way to test it would be to drive the whole + # route and read the answer — which is how a rule ends up asserted by a grep instead of a call. + # ⛔ `dropped` IS NOT NORMALISED AND THAT IS DELIBERATE: it is `[{kind, keys}]`, an action kind + # from OUR catalog and config keys from OUR allowlist, never a model's sentence. The first + # version of this block mapped the normaliser over it and turned each dict into the string + # `"{'kind': ..., 'keys': [...]}"` — `verify_automation`'s own W33 section caught it with a + # `TypeError` on `d["kind"]`. Applying a text rule to a structure is how a channel stops + # carrying what its reader expects, and the reader here is a person's list of what changed. + shown = [dict(r, why=_draft_no_dashes(r.get("why"))) for r in shown] + _notes = [_draft_no_dashes(n) for n in _notes] + return {"draft": {"name": _draft_no_dashes(draft.get("name")) or "New automation", + "trigger": _trig or "manual", + "table": draft.get("table") or "", + "actions": shown}, + "provider": provider, "dropped": dropped, "notes": _notes, "saved": False} + + +# ══════════════════ WAVE 36 · W36-T38 (owner item 8, ruling R9) — AGENT-AUTHORED ACTIONS ══════ +# +# Owner, verbatim (2026-08-18): *"Create the ability for an automation agent to create any +# 'Action' under the Canvas, so its configuration can only be touched by the agent. Be it a tool a +# script etc. We need to really guardrail the reach of this script."* +# +# ⭐⭐ R9 IS A **WRITE** RULE, NOT A VISIBILITY RULE, and the ticket says so in as many words: the +# user MAY read an agent-authored action's configuration and MAY delete it; what they may not do +# is hand-edit it. Hiding a config from the person whose workspace it runs in is a different +# product, and a worse one — nobody can audit what they cannot see. +# +# ⛔ AND IT IS ENFORCED AT THE DOOR, NEVER IN A `disabled` ATTRIBUTE. `routes_automation` already +# carries that lesson for a different control; a client-side lock is a suggestion, and the whole +# point of R9 is that the agent's configuration stays coherent with what the agent believes it +# built. +# +# ⚠ WHY THE MARK IS A SEPARATE BUCKET RATHER THAN A KEY ON THE ACTION. `clean_actions` builds every +# action KEY BY KEY from an allowlist and drops anything it does not recognise (D-75) — so a +# marker stored inside the action would be silently erased on the next save, and the wall would +# quietly stop existing with every gate still green. This is an ANNOTATION layer keyed by +# `(automation id, action id)`; the configuration itself stays where it always was, in the +# automation, with exactly one writer. +AGENT_ACTIONS_KEY = "automation_agent_actions" + +#: Ids this door mints. ⚠ It must match `clean_actions`' `act_[a-z0-9_]{1,32}` or the engine +#: re-mints it and the annotation points at an action that no longer carries that id. +_AGENT_ACTION_PREFIX = "act_ag" + + +def _agent_marks(rt): + """`{auto_id: {action_id: mark}}` for one tenant. `{}` on any failure — an unreadable + annotation must degrade to "nothing is agent-owned", never to a 500 on the automations rail. + + ⛔ AND "DEGRADE TO NOTHING IS OWNED" IS THE SAFE DIRECTION HERE, which is worth stating because + it usually is not. The wall this feeds protects the AGENT's coherence, not the tenant's data: a + lost mark lets a person edit a config they own anyway, in their own workspace, on a step they + could always have deleted outright. A wall that failed CLOSED would instead make an automation + permanently unsavable because one annotation read timed out. + """ + try: + found = rt.get(AGENT_ACTIONS_KEY) or {} + except Exception: # noqa: BLE001 + return {} + return found if isinstance(found, dict) else {} + + +def _flat_actions(actions, out=None, depth=0): + """Every action in a flow, INCLUDING the ones nested inside a group's arms. + + ⚠ NESTED ACTIONS ARE THE ONES THIS MUST NOT MISS. `unconfigured_actions` walks them for the + same reason: a step inside an If/then branch is exactly the one a person cannot see, and a + wall that only looked at the top level would leave the agent's own nested step editable. + """ + out = [] if out is None else out + if depth > 6 or not isinstance(actions, list): + return out + for action in actions: + if not isinstance(action, dict): + continue + out.append(action) + for key in ("then", "else", "actions"): + _flat_actions(action.get(key), out, depth + 1) + return out + + +def _flow_actions(defn): + return _flat_actions(((defn or {}).get("flow") or {}).get("actions") or []) + + +def _same_action(left, right, rt): + """Do these two actions carry the SAME kind and configuration? + + ⛔ COMPARED AFTER `clean_actions`, ON BOTH SIDES, and that is the difference between a wall and + a nuisance. The stored action has already been through the cleaner; a client round-trip has + not, so it may carry a key order, a blank string or a dropped condition that means nothing. + Comparing raw shapes would refuse an edit that changes nothing, and a wall that fires on a + no-op is one an operator learns to route around [[one-question-two-normalizers]]. + """ + def _clean(action): + cleaned, error = engine.clean_actions([dict(action or {}, id="act_1")], rt=rt) + if error or not cleaned: + return None + one = dict(cleaned[0]) + one.pop("id", None) + return one + + a, b = _clean(left), _clean(right) + return a is not None and a == b + + +def _agent_owned_guard(session, auto_id, body): + """R9 AT THE DOOR: a user may not change an agent-authored action's kind or configuration. + + Three outcomes, and the middle one is the ruling: + * the action is ABSENT from the incoming flow -> a DELETE, and R9 allows it + * the action is present and CHANGED -> **409**, naming the action and the agent + * the action is present and identical -> nothing happens, so an ordinary save of a + flow that merely CONTAINS an agent action + is not refused + """ + marks = _agent_marks(session.runtime).get(str(auto_id)) or {} + if not isinstance(marks, dict) or not marks: + return + incoming = (body or {}).get("flow") + if not isinstance(incoming, dict) or "actions" not in incoming: + return + stored = (engine.all_definitions(session.runtime) or {}).get(str(auto_id)) or {} + was = {str(a.get("id") or ""): a for a in _flow_actions(stored)} + now = {str(a.get("id") or ""): a + for a in _flat_actions(incoming.get("actions") or [])} + for action_id, mark in marks.items(): + action_id = str(action_id) + if action_id not in now or action_id not in was: + continue # removed, or never stored: not an EDIT + if _same_action(was[action_id], now[action_id], session.runtime): + continue + who = str((mark or {}).get("agentName") or (mark or {}).get("agent") or "an agent") + raise err(409, "agent_owned", + f"this step was built by {who} and only {who} can change how it is set up. " + f"You can read it, and you can delete it, but it cannot be edited by hand") + + +def _prune_marks(session, auto_id): + """Drop annotations whose action is gone, and the whole entry when the automation is. + + ⚠ CALLED AFTER EVERY WRITE, because the alternative is an annotation bucket that only ever + grows: a mark on a deleted action would keep refusing an id nobody can see, and a mark on a + deleted automation would sit in the tenant document forever. + """ + auto_id = str(auto_id) + + def _set(cur): + cur = dict(cur or {}) + marks = cur.get(auto_id) + if not isinstance(marks, dict): + cur.pop(auto_id, None) + return cur + defn = (engine.all_definitions(session.runtime) or {}).get(auto_id) + if defn is None: + cur.pop(auto_id, None) + return cur + alive = {str(a.get("id") or "") for a in _flow_actions(defn)} + kept = {k: v for k, v in marks.items() if str(k) in alive} + if kept: + cur[auto_id] = kept + else: + cur.pop(auto_id, None) + return cur + + session.runtime.update(AGENT_ACTIONS_KEY, _set, flush="sync") + + +@router.post("/automations/{auto_id}/agent-actions") +def agent_author_action(auto_id: str, body: dict = Body(default=None), + session: Session = Depends(_GATE)): + """THE AGENT'S DOOR: add or replace one Action under the Canvas, marked agent-owned (R9). + + {agent: "", action: {kind, config, when?, id?}} + -> {automation, actionId, agent} + + ⛔ THE ACTION GOES THROUGH `clean_actions` LIKE EVERY OTHER ONE, and that is what "guardrail + the reach of this script" means in code: an agent cannot invent a config key, cannot name a + kind that is not in the catalog, and cannot reach a kind this tenant is not entitled to. The + agent gets a different WALL on editing, never a wider vocabulary. + + ⛔ AND IT CANNOT AUTHOR A KIND THAT IS NOT BUILT. `run_script` is `ready: False` in the + catalog, so `clean_actions` refuses it here exactly as it refuses it for a person — see this + lane's mailbox for why the script ARM is booked rather than half-built: a per-row script + contract and a client card are both missing, and a step that reports success and does nothing + is the failure this repo has already paid for. + + ⚠ THE AGENT ID IS CHECKED AGAINST THE TENANT'S OWN AGENTS. A mark naming an agent that does + not exist would refuse every future edit with a sentence naming nobody. + """ + import routes_slack # noqa: PLC0415 + + body = body if isinstance(body, dict) else {} + agent_id = str(body.get("agent") or "").strip() + agent = routes_slack._agents(session.runtime).get(agent_id) + if not isinstance(agent, dict): + raise err(404, "no_agent", "there is no agent with that id in this workspace") + action = body.get("action") + if not isinstance(action, dict) or not str(action.get("kind") or "").strip(): + raise err(400, "no_action", "an action needs a kind") + + defn = (engine.all_definitions(session.runtime) or {}).get(str(auto_id)) + if defn is None: + raise err(404, "unknown_automation", "no automation with that id") + if str(defn.get("system") or "").strip(): + raise err(409, "system_agent", engine.system_agent_refusal(str(defn["system"]))) + + existing = list(((defn.get("flow") or {}).get("actions") or [])) + taken = {str(a.get("id") or "") for a in _flat_actions(existing)} + action_id = str(action.get("id") or "").strip() + if not action_id or action_id not in taken: + n = 1 + while f"{_AGENT_ACTION_PREFIX}{n}" in taken: + n += 1 + action_id = f"{_AGENT_ACTION_PREFIX}{n}" + fresh = dict(action, id=action_id) + + # ⚠ VALIDATED BEFORE ANYTHING IS WRITTEN, so a refusal leaves the automation exactly as it was. + checked, error = engine.clean_actions([fresh], rt=session.runtime) + if error or not checked: + raise err(400, "invalid_action", error or "that action could not be built") + + replaced = False + for i, one in enumerate(existing): + if isinstance(one, dict) and str(one.get("id") or "") == action_id: + existing[i], replaced = fresh, True + break + if not replaced: + existing.append(fresh) + + notes = [] + updated, error = engine.patch(session.runtime, str(auto_id), + {"flow": {**(defn.get("flow") or {}), "actions": existing}}, + username=session.uname, notes=notes) + if error: + raise err(400, "invalid_automation", error) + + def _set(cur): + cur = dict(cur or {}) + marks = dict(cur.get(str(auto_id)) or {}) + marks[action_id] = {"agent": agent_id, + "agentName": str(agent.get("label") or agent.get("channelName") + or agent_id), + "created": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "by": session.uname} + cur[str(auto_id)] = marks + return cur + + session.runtime.update(AGENT_ACTIONS_KEY, _set, flush="sync") + _prune_marks(session, auto_id) + return {"automation": _wire(updated, session.tenant, rt=session.runtime), + "actionId": action_id, "agent": agent_id, "notes": notes} + + +@router.get("/automations/{auto_id}") +def get_automation(auto_id: str, session: Session = Depends(_GATE)): + """One automation, by id. + + ⛔⛔ WAVE 34 · CONTRACT C3 — A SYNTHETIC ROW MUST BE FETCHABLE BY ID, NOT ONLY LISTABLE, AND + THIS IS THE HALF THAT IS EASY TO SHIP BROKEN. `list_automations` and this route are SEPARATE + lookups: a row appended only to the list appears in the rail, looks entirely real, and 404s + the instant somebody clicks it. `W34-T48`'s `done-when` is *"opening it shows a Trigger and + one step"* — i.e. this route, not the list — so the field agents are resolved here too, from + the same builder, and a person cannot reach a row the detail route does not know. + ⚠ THE SAME BUILDER, NOT A SECOND ONE. `_field_agent_rows` is called and the id looked up in + its output rather than re-deriving one row from the field: two derivations of one row is how + the list and the editor start disagreeing about a name. + """ + if str(auto_id) == ODOO_SYNC_ID: + # ⚠ `detail=True` — the keychain label and the last-sync stamp cost a store read each and + # are resolved HERE, on a route that fetches one automation, never on the list. + row = _odoo_sync_row(session, detail=True) + if row is None: + raise err(404, "unknown_automation", "no automation with that id") + return {"automation": row} + if str(auto_id).startswith("field:"): + row = next((r for r in _field_agent_rows(session) if r.get("id") == str(auto_id)), None) + if row is None: + # ⚠ The SAME 404 as any other unknown id. A field agent whose column has been deleted + # is genuinely gone, and inventing a different error for it would make a normal + # outcome look like a fault. + raise err(404, "unknown_automation", "no automation with that id") + return {"automation": row} + defn = engine.all_definitions(session.runtime).get(str(auto_id)) + if defn is None: + raise err(404, "unknown_automation", "no automation with that id") + return {"automation": _wire(defn, session.tenant, rt=session.runtime)} + + +@router.post("/automations") +def create_automation(body: dict = Body(default=None), session: Session = Depends(_GATE)): + """Create an automation. ⭐ WAVE 23 (C2): the body may carry + `target: {mode: "existing"|"new"|"automated", table?, label?}` — the wizard's FIRST question, + answered before the kind. `new` mints the blank database in the same call, so the automation + is never saved pointing at a table that does not exist yet.""" + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable. Nothing was saved") + # ⭐⭐ D-75 — THE DISCLOSURE CHANNEL, CONSUMED. `clean_actions` silently drops a + # `create_record` condition (D-65's law: dropping is recoverable, refusing locks the door) and + # any config key an arm's allowlist does not keep. Both are deliberate; what was missing was + # anybody being TOLD, so a person saved a flow and got a different one with nothing said. + # ⚠ It rides the SAVE's own response, beside `unconfigured`, because that is the moment the + # person is looking — a note in a log they never open is the same silence with extra steps. + _notes = [] + defn, error = engine.create(session.runtime, body or {}, username=session.uname, + notes=_notes) + if error: + raise err(400, "invalid_automation", error) + return {"automation": _wire(defn, session.tenant, rt=session.runtime), "notes": _notes} + + +@router.patch("/automations/{auto_id}") +def patch_automation(auto_id: str, body: dict = Body(default=None), + session: Session = Depends(_GATE)): + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable. Nothing was saved") + # ⛔⛔ WAVE 34 · CONTRACT C3 — A SYNTHETIC ROW IS NOT PATCHABLE, AND IT MUST SAY SO. A field + # agent is DERIVED from a column definition; there is nothing in the automations bucket to + # write. `engine.patch` would answer "no such automation" (a 404 that reads as "your agent + # vanished") or, worse for a future id shape, find nothing and report success. The refusal + # names where the setting actually lives, because a wall that does not say what to do instead + # sends somebody looking for a bug in a decision made on purpose (D-65's own lesson). + if str(auto_id).startswith("field:"): + raise err(409, "system_agent", + "this agent is an AI enrichment column. Change its prompt, model or schedule on " + "the column itself and this page follows") + if str(auto_id) == ODOO_SYNC_ID: + return _patch_odoo_sync(session, body or {}) + # D-75, same channel as `create` above — an EDIT is where this matters most, because the + # person has just typed the thing that gets dropped. + # ⛔⛔ W36-T38 / R9 — THE AGENT-OWNED WALL, AT THE DOOR, BEFORE ANY WRITE. A `disabled` + # attribute in the builder is a suggestion; this is the refusal. It permits a DELETE of the + # step and permits an ordinary save of a flow that merely contains one. + _agent_owned_guard(session, auto_id, body or {}) + _notes = [] + defn, error = engine.patch(session.runtime, auto_id, body or {}, username=session.uname, + notes=_notes) + if error: + raise err(400 if error != "no such automation" else 404, + "invalid_automation" if error != "no such automation" else "unknown_automation", + error) + # An action the user removed takes its annotation with it, or the mark outlives its subject + # and refuses an id nobody can see. + _prune_marks(session, auto_id) + return {"automation": _wire(defn, session.tenant, rt=session.runtime), "notes": _notes} + + +@router.delete("/automations/{auto_id}") +def delete_automation(auto_id: str, session: Session = Depends(_GATE)): + """Delete the DEFINITION. ⚠ The database it filled is NOT touched — an automation is the + thing that writes rows, not the thing that owns them, and deleting a job must never be a way + to lose data (the same rule the orphan count encodes). + + ⭐⭐ WAVE 34 · CONTRACTS C3 + C4 — A SYSTEM AGENT REFUSES, LOUDLY, AND THE REFUSAL SAYS WHERE + TO GO INSTEAD. Two families are undeletable and they are undeletable for different reasons, so + the sentence names the reason rather than saying "no": a FIELD AGENT is a column (delete the + column), and the ODOO SYNC is the connector's own schedule (disconnect the connector). + + ⛔ REFUSED, NOT IGNORED, AND THAT IS THE POINT OF THE FIRST BRANCH. A synthetic id is not in + the automations bucket, so `engine.remove` would find nothing, do nothing, and this route + would answer `200 {"deleted": ...}` — a delete that reports success and changes nothing, which + is the `view_upsert` failure mode (200 OK, zero writes) arriving in a new place. The row would + then reappear on the next poll and read as a bug in the rail. + """ + # ⛔⛔ EVERY SYNTHETIC ID, NOT JUST THE FIELD ONES. The first version of this guard listed + # `field:` alone, and `verify_automation`'s own C4 leg caught what that left open: deleting + # `system:odoo_sync` fell straight through to `engine.remove`, which popped a key that was + # never in the bucket and answered `200 {"deleted": ...}`. A delete that reports success and + # changes nothing, on the one row the ruling says must be undeletable — the exact defect the + # guard exists for, one id shape away. Both prefixes are refused by the same branch now. + if str(auto_id).startswith("field:"): + raise err(409, "system_agent", engine.system_agent_refusal(SYSTEM_FIELD_AGENT)) + if str(auto_id) == ODOO_SYNC_ID: + raise err(409, "system_agent", engine.system_agent_refusal(SYSTEM_ODOO_SYNC)) + if engine.running(session.tenant, auto_id): + raise err(409, "automation_running", "it is running. Wait for it to finish") + # ⚠ READ BEFORE THE REMOVE, not after: `engine.remove` is the thing being refused. + _defn = (engine.all_definitions(session.runtime) or {}).get(str(auto_id)) or {} + _sys = str(_defn.get("system") or "").strip() + if _sys: + raise err(409, "system_agent", engine.system_agent_refusal(_sys)) + engine.remove(session.runtime, auto_id) + # R9's other half: the user MAY delete an agent-authored step, and deleting the whole + # automation takes its annotations with it rather than stranding them in the document. + _prune_marks(session, auto_id) + return {"deleted": str(auto_id)} + + +@router.post("/automations/preview") +def preview_source(body: dict = Body(default=None), session: Session = Depends(_GATE)): + """What does this URL actually offer? The field-map step — never writes anything.""" + body = body or {} + url = str(body.get("url") or "").strip() + if not url: + raise err(400, "no_url", "give a page URL to read") + try: + return engine.preview(url, str(body.get("extract") or "table"), + int(body.get("tableIndex") or 0)) + except engine.Refused as e: + raise err(400, "refused_url", str(e)) + except Exception as e: # noqa: BLE001 + raise err(502, "fetch_failed", f"could not read that page. {type(e).__name__}: " + f"{str(e)[:160]}") + + +@router.post("/automations/{auto_id}/run") +def run_automation(auto_id: str, session: Session = Depends(_GATE)): + """Start a run on a background thread. 409 when one is already in flight.""" + defn = engine.all_definitions(session.runtime).get(str(auto_id)) + if defn is None: + raise err(404, "unknown_automation", "no automation with that id") + # ⛔ WAVE 32 · T45 (owner item 10) — REFUSED, NAMING THE ACTION. `run_now` refuses too, for the + # tick and the webhook; this one exists so the person who pressed the button reads the reason + # instead of watching a run start and end with nothing done. The two ask the SAME function, so + # they cannot come to disagree about what "configured" means. + refusal = engine.run_refusal(defn) + if refusal: + raise err(400, "action_unconfigured", refusal) + if not engine.run_async(session.runtime, session.tenant, auto_id, username=session.uname): + raise err(409, "automation_running", "that automation is already running") + return {"started": str(auto_id), "startedAt": time.strftime("%Y-%m-%dT%H:%M:%S")} + + +@router.post("/automations/{auto_id}/nodes/{node_id}/toggle") +def toggle_automation_node(auto_id: str, node_id: str, session: Session = Depends(_GATE)): + """Flip one step on the canvas. The SERVER decides what a node's switch means (see + `engine.NODE_TOGGLES`) — the client only reports which node was clicked. + + A node with no switch answers 400 with the sentence saying why, rather than silently doing + nothing: a control that appears to work and does not is worse than one that refuses. + """ + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable. Nothing was saved") + if engine.running(session.tenant, auto_id): + raise err(409, "automation_running", "it is running. Wait for it to finish") + defn, error = engine.toggle_node(session.runtime, auto_id, node_id) + if error: + raise err(404 if error == "no such automation" else 400, + "unknown_automation" if error == "no such automation" else "node_not_toggleable", + error) + return {"automation": _wire(defn, session.tenant, rt=session.runtime)} + + +@router.post("/automations/{auto_id}/hook/{token}") +async def automation_hook(auto_id: str, token: str, request: Request): + """C3's webhook trigger — the tick-endpoint pattern one level down: unauthenticated BY + DESIGN (the external caller has no cookie), gated on a per-automation token minted when the + trigger was configured, constant-time compared. The tenant is FOUND by the (id, token) + pair — a wrong token answers 403 for every tenant, so the route confirms nothing about + which slugs exist. + + ⭐ WAVE 24 (D-41) — THE BODY IS READ DEFENSIVELY AND IS NEVER A REASON TO REFUSE. + ⛔ It is deliberately NOT declared as `body: dict = Body(...)`, which is the obvious way to + write this and would be a live regression: FastAPI would then VALIDATE the payload, so an + existing caller posting text, form-encoding, an empty body or slightly malformed JSON would + start getting a 422 from a door that has accepted anything since wave 22. A webhook sender is + somebody else's system; we do not get to change what it must send in order to fire a flow. + An unreadable body simply maps nothing — the flow still fires, exactly as it did before. + + `run_in_threadpool` keeps the store I/O off the event loop: `hook_fire` walks every tenant + and may commit a row, and this handler had to become `async` only to read the request body. + """ + from starlette.concurrency import run_in_threadpool + from harness import runtime as _rt + + try: + payload_in = await request.json() + except Exception: # noqa: BLE001 + payload_in = None + + def _fire(): + last = (404, {"error": "unknown_automation", + "message": "no automation with that id and token"}) + for slug in _rt.known_tenants(): + try: + rt = _rt.get_runtime(slug) + except Exception: # noqa: BLE001 + continue + status, payload = engine.hook_fire(rt, slug, auto_id, token, body=payload_in) + if status == 200: + return 200, payload + if status != 404: + last = (status, payload) + return last + + status, payload = await run_in_threadpool(_fire) + if status == 200: + return payload + raise err(status, str(payload.get("error") or "refused"), + str(payload.get("message") or "refused")) + + +@router.post("/automations/tick") +def tick(request: Request, x_aios_tick_token: str = Header(default="")): + """Fire every due schedule, for every tenant. The durable-cron entry point (R5). + + Unauthenticated BY DESIGN and gated on a shared secret instead — an EventBridge rule has no + cookie. Refuses when the secret is not configured (see the module header).""" + want = os.environ.get("AIOS_AUTOMATION_TICK_TOKEN") or "" + if not want: + raise err(403, "tick_disabled", + "AIOS_AUTOMATION_TICK_TOKEN is not configured. The tick endpoint is closed") + got = x_aios_tick_token or request.headers.get("X-AIOS-TICK-TOKEN") or "" + if got != want: + raise err(403, "bad_tick_token", "that token is not valid for this deployment") + started = engine.tick_all() + return {"started": started, "at": time.strftime("%Y-%m-%dT%H:%M:%S")} + + +@router.post("/automations/ig/purge") +def purge_ig_subject(body: dict = Body(default=None), session: Session = Depends(_GATE)): + """D-24 (wave 22): the right-to-erasure door for ONE Instagram subject — walks the + tenant's four `ut_ig_*` tables AND the platform master (R2 pooled a copy there, so a purge + that skipped it would not be erasure). Admin-only: erasure is a compliance act, not a + grid gesture. Answers the per-table removal counts — the one aggregate whose drill is the + rows' ABSENCE.""" + if not session.admin: + raise err(403, "admin_only", "erasing a subject is an admin action") + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable. Nothing was " + "purged") + handle = str((body or {}).get("handle") or "").strip() + if not handle: + raise err(400, "no_handle", "name the Instagram handle to erase") + counts = engine.purge_subject(session.runtime, handle) + return {"handle": handle.lstrip("@").lower(), "removed": counts, + "total": sum(counts.values())} + + +@router.get("/automations/metrics/{table_key}/{field_key}/{row_id}/rows") +def metric_drill(table_key: str, field_key: str, row_id: str, + session: Session = Depends(_GATE)): + """C7's drill: the EXACT master snapshot rows behind one metric cell + ([[no-unverifiable-aggregates]]) — recomputed on ask with the same function that filled + the cell, so the drill can never disagree with the number by construction.""" + import core.user_tables as user_tables + + if not user_tables.may_open(table_key, session.uname, session.admin, st=session.runtime): + raise err(404, "unknown_table", "no such database") + t = engine.ut_get(session.runtime, table_key) or {} + fdef = next((f for f in (t.get("fields") or []) if f.get("key") == field_key), None) + if not fdef or not isinstance(fdef.get("metric"), dict): + raise err(404, "not_a_metric", "that column is not a metric field") + row = (t.get("rows") or {}).get(str(row_id)) + if row is None: + raise err(404, "unknown_row", "that record is not in the database") + import ig_master + url_field = next((f.get("key") for f in (t.get("fields") or []) + if f.get("type") == "url"), "") + handle = engine._table_handle(row, url_field) + bag = fdef["metric"] + series = ig_master.series_for({handle}) if handle else {} + value, rows = engine.metric_value(series.get(handle), bag.get("measure"), + bag.get("window"), bag.get("agg") or "") + return {"table": table_key, "field": field_key, "rowId": str(row_id), "handle": handle, + "measure": bag.get("measure"), "window": bag.get("window"), + "value": value, "rows": rows[:200], + "note": "" if value is not None else + "no master data answers this window. The cell is honestly blank"} + + +@router.get("/automations/{auto_id}/rows") +def run_rows(auto_id: str, session: Session = Depends(_GATE)): + """The rows the LAST run touched — the drill-down behind a run's counts. + + Every count in this product drills to the exact rows behind it ([[no-unverifiable-aggregates]]); + a run history that said "412 updated" and could not show which would be the thing that rule + exists to forbid. + """ + defn = engine.all_definitions(session.runtime).get(str(auto_id)) + if defn is None: + raise err(404, "unknown_automation", "no automation with that id") + last = (defn.get("runs") or [{}])[0] + table_key = (defn.get("config") or {}).get("targetTable") or "" + t = engine.ut_get(session.runtime, table_key) or {} + rows = t.get("rows") or {} + ids = [str(i) for i in (last.get("affected") or [])] + return {"table": table_key, "label": t.get("label") or table_key, + "fields": [{"key": f.get("key"), "label": f.get("label")} + for f in (t.get("fields") or [])], + "rows": [{"id": i, **(rows.get(i) or {})} for i in ids if i in rows], + "truncated": len(ids) >= 200} + + +# --- the in-process scheduler ------------------------------------------------------------ +# ⚠ OPT-IN (`AIOS_AUTOMATIONS=1`), and that is an AMENDMENT to the wave brief's "default-on", +# made on a measurement: `verify_api.py` runs for 196 s, i.e. longer than three tick intervals, +# and `tick_all` reaches `runtime.get_runtime()` — which builds tenants and moves the LRU cache +# that verify_api's own isolation checks read. Default-on would put a background thread inside +# the subject of another session's gate. `AIOS_PREWARM=1` at `main.py:353` is the same decision +# for the same reason, so this follows it rather than inventing a second convention. +# +# The loop also SLEEPS FIRST (`engine.scheduler_loop`) — defence in depth, proven in +# verify_automation.py section T rather than assumed. +# +# ⛔ THE DEPLOY MUST SET `AIOS_AUTOMATIONS=1` (with `AIOS_AUTOMATION_TICK_TOKEN`) or schedules +# only ever fire from the external cron POSTing /automations/tick. Both paths work; neither is +# implicit. Booked in the session-D mailbox. +engine.start_scheduler() + +# C3 (wave 22): register the trigger listener onto the platform's row-event seam. THIS module +# is where the registration belongs — it is the one place that already imports both sides, so +# neither the engine nor platform/core grows a dependency on the other. Idempotent: a reimport +# must not double-fire every trigger. +import core.user_tables as _ut_hooks # noqa: E402 + +if engine.grid_hook not in _ut_hooks.ROW_HOOKS: + _ut_hooks.ROW_HOOKS.append(engine.grid_hook) + +# ⭐⭐ W31 QA — DECLARE THE MACHINE-OWNED CHILD DATABASES, for the same reason and in the same +# place as the hook above: this module already imports both sides, so neither the engine nor +# `platform/core` grows a dependency on the other. Idempotent by construction (a set). +# +# ⛔ THE OWNER FOUND WHAT THIS FIXES, IN PRODUCTION, AFTER THE TICKET READ GREEN. W31-T32 locked +# the TikTok children by stamping `recordMode` at the two SPAWN sites, and proved the stamp +# arrives "on the next write". That is true and it is not the `done-when`: every TikTok database +# already sitting in a tenant kept offering "+ New record" until somebody re-ran a TikTok +# automation, and nobody had. Owner, verbatim (2026-08-13): *"the databases for Tiktok do not have +# the small lock icon as I asked"* — and the rule, restated: *"just like Instagram Post database +# (which is locked), only the IG Profile and TT Profile should be editable."* +# ⚠ A DECLARATION NEEDS NO WRITE, so it is true for EVERY tenant the moment the API boots — no +# migration, no boot ordering, no per-tenant walk, and nothing that a stale store can undo. The +# stored flag still locks a table nobody declares; the two are OR'd. +_ut_hooks.register_locked_records(engine.LOCKED_CHILD_TABLES) diff --git a/api/routes_customers.py b/api/routes_customers.py index ab6ac1a78da96096ce199007ccbabef07de0aa8c..5d1cb02b64e9415c1ede004189b052d452bf8bc6 100644 --- a/api/routes_customers.py +++ b/api/routes_customers.py @@ -1,834 +1,834 @@ -"""routes_customers.py — X2's read + write of the customer table, BU-SCOPED (EXIT-3b / EXIT-2a). - -Two things happen here that did not happen in the pre-wave `main.py`: - - 1. **ROWS ARE SCOPED TO THE SESSION.** The old `/api/customers` served `cl.pool()` — the whole - book, to anyone holding the shared APP_PASSWORD. Now the pool is built with - `(team_id, agent_name)` derived from the USER RECORD, so a Royal-only user never receives a - Fisch row and an agent-linked login never receives another rep's book. The scope is applied - at the QUERY, not as a post-filter, so there is no moment at which the other BU's rows exist - in this response. - - 2. **THE OVERLAY FORK IS GONE.** `aios-web/api/data/overlay.json` was a SECOND writable home - for the same user-owned fields the Streamlit app keeps in the tenant store — two truths, and - whichever process you asked last was right. Reads and writes now both go through - `modules.customer_data`'s table-workspace functions: ONE store (C1c, ARCHITECTURE §1a rule 3). - -⚠ ONE STORE IS NOT YET ONE CACHE (strangler-period, booked honestly). `core.store.get()` is -cache-first per PROCESS, so a write from the Streamlit container is invisible to a running API -container until its cache is refreshed, and vice versa. Deleting the fork removes the second -SOURCE OF TRUTH; it does not make the two runtimes coherent. The fix is X4/Postgres (task C-4, -owner-blocked on B-3), and no test here may claim read-your-writes ACROSS runtimes — a TestClient -proof is single-process and would report green on exactly the thing that is still broken. -""" -import time - -from fastapi import APIRouter, Body, Depends - -import scope_cache -from deps import Session, err, module_gate, perms - -router = APIRouter(prefix="/api/v1") - -#: The surface these routes serve. Both legs are gated on it, so a user without the grant gets a -#: 403 rather than an empty table that looks like "you have no customers". -MODULE = "customer_data" - -_CACHE_TTL = 900 # the pool build is slow (Odoo + reconciliation); 15 min, as before - - -def _pool_rows(session: Session): - """The reconciled Odoo pool for this session's SCOPE — the slow part, and the only part that - may be shared between users. - - ⛔ WHAT MAY BE CACHED HERE, AND WHY THE LINE IS EXACTLY HERE. The cache key is - `(team_id, agent)` and the cached value is the raw pool: Odoo-source columns only. That is - genuinely scope-shaped — two users with the same BU and the same book are asking the same - question, and `cl.pool()` is expensive (an Odoo pull plus reconciliation). - - The FULL PAYLOAD is NOT cacheable on this key, and caching it here was a real defect I shipped - and then removed. `fields` comes from `fields_from_workspace(ws)` — that user's own `custom_` - and `measure_` columns — and every overlay cell comes from `ws['overlays']`; the table - workspace is read as `data[username]` and written as `patch_table_overlay(uname, …)`, i.e. - PER USER. So two Royal-only users with no agent link share `(6, None)` and the second one - would have been served the FIRST one's private notes and private columns. The scope key was - right for the pool and wrong for everything wrapped around it. - - It survived a green 129-check battery because every fixture user had a DISTINCT - `(team_id, agent)` pair, so no two of them ever collided on the key — the test set could not - express the bug. `verify_api.py` now carries a same-scope second user for exactly this. - """ - rt = session.runtime - team_id, agent = _team_agent(session) - return _pool_for(rt, team_id, agent) - - -def _pool_for(rt, team_id, agent): - """The cached pool for an explicit scope — session-free so the prewarm thread and the - stale-refresh path can call it. STALE-WHILE-REFRESH (scope_cache): once a copy exists no - request blocks on the 10–30s Odoo rebuild again; only a scope's FIRST-ever build does. - - DEBT-2 (2026-08-04): while the tenant's RESOLVED Odoo source is PAUSED this path never - goes live — it serves the in-process copy at any age, else the persisted pause-time - snapshot (restart-safe), else answers 503. Never a WIDER scope's snapshot: handing a - scoped user the consolidated rows would widen their book, which is worse than an error.""" - import modules.customer_data as cl - import routes_keychain - - key = ("pool", team_id, agent) - - if routes_keychain.odoo_paused(rt): - hit = rt.pool_cache.get(key) - if hit: - return hit[1] - snap = routes_keychain.load_pool_snapshot(rt, team_id, agent) - if snap is not None: - rt.pool_cache[key] = snap # seed, so the memo stamps stay coherent - return snap[1] - raise err(503, "connector_paused", - "this data source is paused and no snapshot exists for your scope — " - "an admin can resume it under Settings → Connectors") - - def _build(): - # ⚠ THE SCOPE GOES INTO THE BUILDER. `pool(agent_name, team_id)` is the same reconciled - # builder the Streamlit page uses — passing the scope here is what makes the isolation a - # property of the QUERY instead of a filter someone can forget to apply downstream. - return cl.pool(agent, team_id) - - def _evict(): - # Bounded: a scope cache that only ever grows is a memory leak in a shared process. - if len(rt.pool_cache) > 16: - for stale in sorted(rt.pool_cache, key=lambda k: rt.pool_cache[k][0])[:8]: - rt.pool_cache.pop(stale, None) - - return scope_cache.get(rt.pool_cache, key, _CACHE_TTL, _build, _evict) - - -def warm_default(rt): - """Boot prewarm: the consolidated pool `(None, None)` — the scope every admin and every - all-BU account lands on. Called from main.py's prewarm thread only.""" - _pool_for(rt, None, None) - - -def _team_agent(session: Session): - """The `(team_id, agent)` this session's POOL is built with — ONE derivation point, used by - `_pool_rows` and `grid_assembly` alike, so the cache key and the query can never disagree. - - ⛔ WAVE 15 R1: THIS NOW COMES FROM THE PERMANENT FILTER (C-PERM amendment 3). `team_id` is - not a row filter — `customer_data._pool_build` passes it into `cust._cust_rev` three times - and into `_cadence_bulk`, so it decides what `rev`/`ly`/`ltm`/`aov`/`status` MEAN. Enforcing - a BU purely as a post-filter would keep the row list right and silently consolidate every - number. So the pushdown survives as a DERIVATION OF the declared filter rather than a second - wall beside it, and `perm_scope.derive_pool_scope` falls back to the legacy `bus`/`agent` - derivation for any record the migration has not reached yet. - """ - import core.perm_scope as perm_scope - return perm_scope.derive_pool_scope(session.user, MODULE) - - -def _pool_stamp(rt, team_id, agent): - """The cached pool's build timestamp — the DATA STAMP in every measure-memo key, so a pool - refresh invalidates the memoised answers exactly when the underlying rows changed.""" - entry = rt.pool_cache.get(("pool", team_id, agent)) - return entry[0] if isinstance(entry, tuple) and entry else 0 - - -def _measure_err(tag, e): - try: - import harness.telemetry as _tel - _tel.error(f"api:{tag}", e) - except Exception: - pass - - -# ══════════════════════════ THE TENANT-WIDE STRATUM, ON THE CUSTOMER TOPIC (W38-T20 / D-425) ══ -# -# ⛔⛔ WHY THIS FILE GREW A SHARED STRATUM AT ALL. `core/shared_overlay.py` has been generic since -# W29-T62, `routes_products.py` has merged it since W30-T36 and `routes_tables.py` since W38-T16 — -# and the CUSTOMER topic had neither a write door nor a read merge. Every user-created column here -# lives in `data[username]`, so two accounts looking at "the same" column are looking at two -# columns. That is fine for a private note and fatal for a ROUTE ORDER: a visit sequence one rep -# can see and their colleague cannot is not a plan, it is a rumour. -# -# ⛔ ONE SPELLING OF THE BUCKET. `modules.customer_data.TABLE_KEY` is the per-user workspace key -# and `shared_overlay.bucket()` derives `__shared` from it. Resolving it here rather than -# writing the string means the write door and the read merge cannot disagree about where the -# values live — which is exactly the failure T16 found on the materialised `ut_*` tables, where -# `patch_shared_cell` wrote into a bucket no reader ever opened. - - -def _shared_key(): - """The store key this topic's per-user AND tenant-wide strata are both named from.""" - import modules.customer_data as cl - return cl.TABLE_KEY - - -def shared_fields(st=None): - """`{field_key: Field}` — the columns this topic shares tenant-wide. - - Unscoped on purpose, exactly as `shared_overlay.fields` is: a shared column's EXISTENCE is - tenant-wide by definition. WHO MAY SEE IT is a separate question, answered one layer up by - `perm_scope.hidden_keys` (the per-field grant wall T16 landed), and WHOSE ROWS by `cells`. - """ - from core import shared_overlay - try: - return shared_overlay.fields(_shared_key(), st=st) - except Exception: # noqa: BLE001 - # Lenient like every other display read: an unreachable store degrades to "nothing is - # shared yet", never to a 500 on a grid that would otherwise render. The WALL does not - # degrade with it — `field_grant_hidden` hides a marked column it cannot resolve. - return {} - - -def shared_cells(pids, st=None): - """`{"": {key: value}}` for the rows named by `pids`, and ONLY those. - - ⛔ `pids` IS THE ROW WALL, PASSED AND NEVER DEFAULTED. `shared_overlay.cells` refuses an - "everything" read by signature for this reason; the set handed in is the one `grid_assembly` - has already narrowed with `apply_row_scope`, so a cell belonging to the other BU has nothing - to attach itself to. - """ - from core import shared_overlay - try: - return shared_overlay.cells(_shared_key(), list(pids or ()), st=st) - except Exception: # noqa: BLE001 - return {} - - -def _merge_shared_fields(fields, defs): - """`fields` PLUS the tenant-wide columns this topic declares — `routes_tables._ut_shared_fields` - on the customer topic. - - ⚠ MERGED BEFORE THE WALL, NEVER AFTER. `hidden_keys` is a TRANSITIVE closure, so it must run - on the WHOLE contract: a formula over a shared column that reads a hidden one sits outside the - closure's reach otherwise and carries the hidden value out wearing a second name. It is also - the only order in which `field_grant_hidden` can ever see the `granted` marker at all — merge - afterwards and the per-field wall is inert while every test still passes. - ⚠ A key the canonical contract already declares WINS. A shared column is an ADDITION to this - database's contract, never a redefinition of a column it already has. - """ - if not defs: - return fields - have = {f.get("key") for f in (fields or ()) if isinstance(f, dict)} - return list(fields or ()) + [dict(f, source="overlay") - for k, f in defs.items() if k not in have] - - -def grid_assembly(session: Session, scope: str = "customer", storage_key: str = "", - consume_corrections: bool = True): - """ONE assembly of this session's grid state, shared by `/customers`, `/workspace` and the - events route (2026-07-31 — the standalone measure gap, owner item 1). - - What it adds over the pre-wave hand-rolled `_payload` loop, and why it replaced it: - - * rows go through `aios_grid.rows_from_pool` — the SAME builder the embedded host uses, - so `lat`/`lon` (the Map view's data), `_created` and every derived column ride each row - by construction instead of by a second loop that drifts. The hand loop was written to - mirror the pre-Map contract and silently dropped the coordinates: the shell's Map view - had nothing to plot ("the Map no longer works"). - * `derived` carries the cohort column's cells AND the measure columns' values, resolved - through `core.measure_resolve` (EXIT-5's extraction of the `_cl_measure_*` family). - Without them every measure column the owner built rendered BLANK in the shell. - * `measures` (the offer) and `measure_sets` (condition answers) are computed here so the - events route can finally validate measure fields/conditions instead of refusing them - (an empty `measure_offer` made `clean_measure_field` reject every create over HTTP). - - Memos live on the TENANT RUNTIME (`rt.measure_memo` / `rt.mset_memo`) — bounded by the - module's own clear-past-cap rule, keyed on (stamp, scope, pool identity, question), nothing - user-shaped in them. - """ - import aios_grid - from core import grid_events, measure_resolve - - import core.perm_scope as perm_scope - - rt = session.runtime - team_id, agent = _team_agent(session) - rows_src = _pool_for(rt, team_id, agent) - # ⛔ THE ROW WALL, APPLIED BEFORE `pids` IS TAKEN. Everything downstream is bounded by that - # frozenset — `allowed_pids` for the workspace, cohort membership, measure resolution — so - # scoping here means a row this account may not see never enters ANY of them, rather than - # being filtered out of one payload and surviving in another. - # - # Evaluated against the CANONICAL field list, not the per-user assembled one, for two - # reasons: the assembled list is not built yet (it needs `pids`), and a permanent filter may - # only ever name a canonical field anyway — `routes_admin._clean_perms` validates it against - # exactly this schema and 400s otherwise. `permits()` denies on anything it cannot answer. - rows_src = perm_scope.apply_row_scope(rows_src, session.user, MODULE, aios_grid.FIELDS) - pids = frozenset(r["pid"] for r in rows_src if r.get("pid") is not None) - # ⭐ W38-T20 — THE COLUMN DEFINITIONS ARE READ **ONCE** PER ASSEMBLY AND THREADED, because - # `core.store.get` deep-copies whatever it hands back on every call. Three consumers want this - # dict on one request (the write ctx's wall, the read merge, and the closure), and letting each - # take its own copy is the shape D-214 spent a whole ticket removing one document over. - _defs = shared_fields(st=rt) - ws = grid_events.table_workspace( - _ctx_for(session, pids, defs=_defs), allowed_pids=pids, - consume_corrections=consume_corrections) - # ⭐⭐ W38-T20 / D-425 — THE TENANT-WIDE CELLS, LAYERED OVER THE PER-USER ONES, IN THE - # ASSEMBLY SO EVERY CONSUMER SEES ONE TRUTH. `routes_grid`'s /workspace route serves - # `workspace["overlays"] = g["ws"].get("overlays")` verbatim and `_payload` hands the same - # dict to `rows_from_pool`, so merging HERE reaches both without touching either file. - # - # ⚠ SAFE TO MUTATE, and checked rather than assumed (the same check `product_assembly` - # records): `table_workspace` reads through `store.get`, which deep-copies, so `ws` is a - # detached copy and nothing writes it back. A shared value can never leak INTO the per-user - # bucket by way of this merge. - # ⚠ SHARED WINS PER KEY. The whole point of the stratum is that every reader sees the same - # number, so a per-user leftover under the same key is stale by construction. It is also what - # makes D-423 recoverable rather than permanent: a pre-fix per-user edit is shadowed, not - # promoted. - _shared = shared_cells(pids, st=rt) - if _shared: - _ov = dict(ws.get("overlays") or {}) - for _pid, _cells in _shared.items(): - _ov[_pid] = {**(_ov.get(_pid) or {}), **_cells} - ws["overlays"] = _ov - workspace, fields, views, lists = aios_grid.workspace_wire( - ws, session.uname, set(pids), defs={}, scope_key=scope, storage_key=storage_key) - # ⭐⭐ W38-T20 — AND THE COLUMN DEFINITIONS, BEFORE THE WALL. See `_merge_shared_fields`: this - # position is load-bearing twice, once for the transitive closure and once because it is the - # only order in which the per-field grant marker is ever presented to `hidden_keys`. - fields = _merge_shared_fields(fields, _defs) - # THE FIELD WALL — a TRANSITIVE closure (C-PERM amendment 5), so hiding a field also hides - # every formula computed FROM it. Formulas evaluate in the browser from `{ref}`s, so - # shipping a dependent formula while withholding its input either leaks the input through - # the formula's value or silently computes a wrong one; only removing both is coherent. - # Applied AFTER workspace_wire because custom + measure columns are what it must cover. - # ⭐ W38-T20 — `st=rt` IS NOT TIDINESS. `perm_scope.field_grant_hidden` resolves a marked - # column against `object_shares`, and without a tenant handle it reads the module-default - # bucket: on any tenant but #0 that finds no grant, and no grant on a MARKED column means - # HIDDEN. So an unthreaded `st` UNDER-shares (a grantee cannot see their own column) rather - # than over-shares — visible and reportable, but still wrong, and `visible_fields`' own note - # requires the two calls to agree about it. - hidden = perm_scope.hidden_keys(session.user, MODULE, fields, st=rt) - if hidden: - fields = [f for f in fields if f.get("key") not in hidden] - # The field LIST and the ROW payload are two different wires. Narrowing only the first - # would leave the value sitting in the second, where anything can read it. - rows_src = [perm_scope.strip_row(r, hidden) for r in rows_src] - # ⛔⛔ AND THE OVERLAY DICT IS A THIRD WIRE, WHICH IS NEW THIS TICKET AND WAS A HOLE - # BEFORE IT. `rows_from_pool` iterates `fields`, so a narrowed contract already keeps a - # hidden key off a ROW — but `/workspace` serves `ws["overlays"]` RAW, and that dict is - # where both the per-user cells and (as of the merge above) the tenant-wide ones sit. - # One narrowing cannot speak for a wire it never touches; `routes_tables` writes the same - # sentence over its own `shared_cells`, and `routes_odoo_tables` over its `overlays`. - # ⚠ `ws` AND NOT `workspace`: `routes_grid.workspace` copies the dict ACROSS - # (`workspace["overlays"] = g["ws"].get("overlays")`) after this returns, so narrowing the - # source is what reaches the wire. Narrowing the copy would be narrowing a key that gets - # overwritten a moment later. - _ov = ws.get("overlays") or {} - if _ov: - ws["overlays"] = {pid: {k: v for k, v in (cells or {}).items() if k not in hidden} - for pid, cells in _ov.items()} - - today = time.strftime("%Y-%m-%d") - stamp = _pool_stamp(rt, team_id, agent) - # ⭐⭐ W38-T19 — THE METRICS CAPABILITY, AND THIS GRAIN NEEDS **THREE** GUARDS WHERE THE - # OTHER TWO NEED ONE. `routes_products` and `routes_tables` funnel their cells and their - # condition answers through helpers that short-circuit on an empty `offer`, so emptying the - # offer there stops the whole feature. `core.measure_resolve` does not take an offer at all: - # `condition_sets` and `column_values` both re-derive their work from the caller's OWN saved - # views and field list. So gating only the offer here would take the Metric kind off the - # picker and refuse new creates while an EXISTING Metric column kept computing and an - # EXISTING measure condition kept resolving — a revoked capability still answering, on the - # grain with the most of them. Three calls, one predicate. - may_metrics = perm_scope.may_metrics(session.user, MODULE) - measures = measure_resolve.offer(team_id, on_error=_measure_err) if may_metrics else [] - measure_sets = measure_resolve.condition_sets( - [v.get("config") or {} for v in (views or [])], None, team_id, pids, today, stamp, - rt.mset_memo, on_error=_measure_err) if may_metrics else {} - # The derived channel: cohort membership cells + measure column values, ONE dict — the - # same read-only channel the embed host hands to rows_from_pool. - derived = aios_grid.cohort_cells(lists) - # ⭐ W33-T43 / owner item 12 ("One unique ID per database always"). The customer grid carried - # NO Odoo id column at all, while its retiring twin `ut_odoo_customers` carried `partner_id` - # as its join key — so the merge would have lost the one value every Odoo document joins on. - # - # ⛔ IT IS DERIVED, NOT A POOL COLUMN, AND THAT IS THE WHOLE POINT: a customer row's `pid` IS - # the `res.partner` id (`modules/customer_data.pool` mints it that way and the identity is - # asserted against Odoo in that module). Adding it to the pool would be a SECOND source for - # one fact, which is the class of defect item 12 is about. This channel exists for exactly - # this — a value the host knows per render and the pool has no business storing. - for pid in pids: - derived.setdefault(pid, {})["partner_id"] = pid - # ⭐⭐ W38-T19 — SKIPPED ENTIRELY WHEN THE CAPABILITY IS REVOKED, rather than filtered after. - # `column_values` selects its own subjects (`isinstance(f.get('measure'), dict)`) off the - # field list, so there is no argument that could narrow it; not calling it is the narrowing. - # ⚠ THE COLUMN STAYS AND ITS CELLS GO BLANK, which is this channel's OWN documented degrade - # ("blank is could not compute, 0 is a real zero") and is what the product and user-table - # grains already do under an empty offer. Deleting the column instead would be a second, - # louder behaviour for the same fact on one surface out of three, and it would destroy a - # definition the admin can restore with one tick. - for pid, cells in (measure_resolve.column_values( - fields, team_id, pids, today, stamp, rt.measure_memo, - on_error=_measure_err) if may_metrics else {}).items(): - derived.setdefault(pid, {}).update(cells) - - return {"rows_src": rows_src, "pids": pids, "ws": ws, "workspace": workspace, - "fields": fields, "views": views, "lists": lists, "derived": derived, - "measures": measures, "measure_sets": measure_sets, "today": today, - "team_id": team_id} - - -def _payload(session: Session): - """`{fields, rows, today, docs, pulled_at}` — X2's shape, which `verify_fields_contract.py` - referees. Rows are now built by `aios_grid.rows_from_pool` (embed == standalone by - construction); see `grid_assembly` for what that fixed. - - ⭐ `docs` joined the shape in wave 30 (W30-T37 / contract C4). Named here rather than left to - the reader because a docstring that still lists the OLD shape is a stale comment on correct - code — this repo's D-73 — and it is the first thing anyone greps to learn the payload. - - ⚠ `rows_src` is the SHARED cached list — `rows_from_pool` reads it and builds NEW dicts, - never mutating a cached row (the same-scope-second-user leak rule). - """ - import aios_grid - - g = grid_assembly(session) - rows = aios_grid.rows_from_pool( - g["rows_src"], g["fields"], g["ws"].get("overlays"), derived=g["derived"]) - # ⭐ C4 / D-138 (W30-T37) — THE DOCUMENTS PRODUCER FOR THE CUSTOMER SCOPE. The write door - # (`doc_add`/`doc_fetch`/`doc_delete`) never stopped working and every client half is - # complete; what vanished with `app.py` at EXIT-6 was the only thing that ever set this key. - # All six `onDoc*` handlers in `CustomerGrid.tsx` read `payload?.docs ? … : undefined`, so an - # ABSENT key — not a broken one — is what has been switching the whole feature off. - # - # ⛔ IMPORTED, NEVER RE-SERIALISED. `core.grid_events.docs_for` is the ONE serialiser and - # `routes_tables` (the `ut_*` scope) calls the SAME function with the same argument order. - # A matching pair here is precisely how the wave-29 close-out reintroduced its own defect in - # the opposite direction inside a single commit ([[one-question-two-normalizers]]). - # ⚠ `g["pids"]` is the row set this session is ALREADY scoped to — `docs_for` has no - # "every document in the tenant" mode to reach for, deliberately. - from core import grid_events as _ge - return {"fields": g["fields"], "rows": rows, - # `today` rides the payload because every relative date condition must resolve against - # the TENANT's day, never the browser's — a client that falls back to its own clock - # disagrees with the server for everyone west of it. - "today": g["today"], - "docs": _ge.docs_for(g["pids"], scope_key="customer", uname=session.uname, - admin=session.admin, st=session.runtime), - "pulled_at": time.strftime("%Y-%m-%d %H:%M")} - - -def _ctx_for(session: Session, pids, defs=None): - """An EventCtx for the READ path — no fallback workspace, so a store outage is a 503 rather - than a phantom in-memory workspace an API request cannot persist. - - `defs` (W38-T20) is this topic's tenant-wide column definitions when the caller already holds - them; absent, they are read. One read per assembly rather than one per question asked of it. - """ - from core import grid_events - return grid_events.EventCtx( - uname=session.uname, allowed_pids=frozenset(pids or ()), fields=[], - # C-PERM: the write wall's field half. Computed from the CANONICAL contract PLUS the - # tenant-wide columns, because `fields=[]` here — the closure only needs the schema, not - # this user's column list, and a runtime column is part of that schema now. - hidden_keys=_hidden_for(session, defs=defs), - admin=session.admin, fallback_ws=None, seen_ids={}) - - - -def _hidden_for(session: Session, defs=None): - """The fields this session's permissions hide — the write wall's half of C-PERM. - - Read paths strip these from both wires so they cannot be SEEN; this is what stops them - being WRITTEN by a caller who knows the key. Evaluated against the canonical contract, the - same schema `routes_admin._clean_perms` validates a hiddenFields entry against. - - ⭐⭐ W38-T20 — AND THE TENANT-WIDE COLUMNS ARE PART OF THAT CONTRACT NOW, WHICH IS A WALL AND - NOT A COMPLETENESS TIDY-UP. `patch_customer` builds its ctx with `fields=payload["fields"]`, - which carries the merged shared columns, so `grid_events.handle_one` would happily accept an - `overlay_patch` naming one. The canonical list cannot mention them (they are created at - runtime), so a wall computed from `aios_grid.FIELDS` alone answers "not hidden" for every - grant-governed column and the write door is open to a reader who was never granted it. - ⚠ Widening the field list can only ever ADD to the hidden set, never remove from it: the - closure hides what it is told to hide plus whatever depends on it. - """ - import aios_grid - import core.perm_scope as perm_scope - - if defs is None: - defs = shared_fields(st=session.runtime) - return perm_scope.hidden_keys( - session.user, MODULE, _merge_shared_fields(list(aios_grid.FIELDS), defs), - st=session.runtime) - -def allowed_pids(session: Session): - """The pids this session may touch — the POOL's own ids, so the write wall and the read scope - can never disagree. - - Reads `_pool_rows` rather than `_payload`: the wall only needs identities, and going through - the full payload would pay for a workspace read and a row assembly on every write. - - ⛔ THE PERMANENT FILTER APPLIES HERE TOO, AND FORGETTING IT IS A WRITE-WITHOUT-READ HOLE. - `_pool_rows` is built with the DERIVED pushdown, which expresses only what a `(team_id, - agent)` pair can express. Any part of the wall the pushdown cannot carry — `revenue > 1000`, - a nested group, a condition on any other column — leaves the pool WIDER than the filter. Read - paths close that gap with `apply_row_scope`; without the same call here the write wall would - be the wider set, and a restricted user could PATCH a row this API will not show them. - Same function, same order as `grid_assembly`, so the two walls cannot drift. - """ - import core.perm_scope as perm_scope - import aios_grid - - rows = perm_scope.apply_row_scope(_pool_rows(session), session.user, MODULE, - aios_grid.FIELDS) - return frozenset(r["pid"] for r in rows if r.get("pid") is not None) - - -@router.get("/customers") -def customers(session: Session = Depends(module_gate(MODULE))): - return _payload(session) - - -@router.patch("/customers/{pid}") -def patch_customer(pid: int, body: dict = Body(default=None), - session: Session = Depends(module_gate(MODULE))): - """Write the EDITABLE overlay stratum only — Odoo stays read-only, forever. - - Routed through `core.grid_events.handle_one` as an `overlay_patch` event rather than writing - the store directly: that handler is where the per-key `permissions.edit` wall, the pid wall - and the truncation rules live, and a second implementation of those would be a second set of - them to keep in step. The response reports what was ACCEPTED, which is not always what was - asked for. - """ - from core import grid_events - - updates = dict(body or {}) - if not updates: - raise err(400, "empty_patch", "no fields to update") - pool = allowed_pids(session) - if pid not in pool: - # 403, not 404: the pid may well exist — it is simply not in this session's book, and - # saying "no such customer" would confirm the opposite to anyone who guessed right. - raise err(403, "out_of_scope", "that customer is not in your book") - payload = _payload(session) - ctx = grid_events.EventCtx( - uname=session.uname, allowed_pids=pool, fields=payload["fields"], - admin=session.admin, fallback_ws=None, seen_ids={}, - hidden_keys=_hidden_for(session)) - try: - grid_events.handle_one( - {"id": f"patch:{pid}:{time.time_ns()}", "type": "overlay_patch", - "pid": pid, "updates": updates}, ctx) - except grid_events.StoreUnavailable: - raise err(503, "store_unavailable", - "the tenant store is unavailable — your change was not saved") - - # What actually landed, read back from the store rather than echoed from the request: a - # refused key or a truncated value must not be reported as accepted. - stored = (grid_events.table_workspace(_ctx_for(session, pool), allowed_pids=None) - .get("overlays") or {}).get(str(pid)) or {} - accepted = {k: stored.get(k) for k in updates if k in stored} - refused = sorted(k for k in updates if k not in accepted or stored.get(k) != str(updates[k])) - # No cache to patch: the overlay stratum is re-read from the store on every `_payload`, so - # read-your-writes within this runtime is a property of the design rather than of a - # write-through step somebody has to remember. (It was a write-through step while the whole - # payload was cached on a scope key — the arrangement that leaked one user's notes to - # another. See `_pool_rows`.) Cross-RUNTIME coherence is still not claimed: the module - # docstring says why, and Postgres is the fix. - out = {"ok": True, "pid": pid, "updates": accepted} - if refused: - out["refused"] = refused - return out - - -# ══════════════════════════════════ THE ROUTE-ORDER COLUMN (W38-T20 / ruling R7 / contract C1) ══ -# -# ⛔⛔ WHY THIS IS A DOOR HERE AND NOT A KIND IN THE COLUMN MENU. `ColumnMenu.onCreate` is the only -# persistence the menu has, and it lands a PER-USER `custom_` column through -# `aios_grid._field_extras`, which is a strict allowlist: a route-order bag created that way is -# silently stripped and its values are private to their author. That is the wave-19 `image` -# failure verbatim ("created, named, configured, gone", recorded in `_clean_geocode`'s own -# docstring) plus a done-when clause that cannot hold, because a colleague reading a per-user -# stratum gets a clean 200 with nothing in it. The `geocode` pseudo-kind is the precedent for the -# PICKER; its persistence half does not transfer. -# -# ⭐ SO THE COLUMN IS BORN SHARED. `shared_overlay.put_field` stores the definition verbatim (no -# allowlist), which is what lets the input fingerprint ride the DEFINITION rather than the rows — -# `shared_overlay._value` RAISES on a dict, so `{order, inputsHash}` could never be one cell, and -# one solve fingerprints the whole cohort identically anyway, so per-row would be the same string -# written N times. -# -# ⭐ AND THE NUMBER ON THE RECORD IS THE INVERSE OF THE PLANNER'S ANSWER. `mapProjection.planRoute` -# returns `order`, where `order[i]` is WHICH STOP is visited i-th; the cell holds the RANK. The -# client inverts it with `routeRanks` (gated in `map.test.ts` at the desktop shape, over a fixture -# chosen so the two differ); this door then refuses anything that is not a clean 1..N, so a -# truncated or double-posted body cannot land as a half-route. - -#: The key prefix every route-order column wears. `custom_` and `measure_` are the two existing -#: created-column namespaces and both are PER USER; this one is tenant-wide, so it takes its own -#: rather than borrowing a prefix whose readers assume a per-user home. -ROUTE_KEY_PREFIX = "route_" - -#: The marker on the stored definition that says WHAT this column is. Read by the GET below and by -#: the client; never inferred from the key, because a prefix is a naming convention and a -#: convention is not a declaration. -ROUTE_KIND = "route_order" - -#: ⛔⛔ THE TOPIC A FIELD GRANT IS NAMED UNDER, AND IT IS **NOT** THE STORE BUCKET. Two namespaces -#: meet on this column and they are spelled differently: -#: -#: the STRATUM lives at `shared_overlay.bucket(customer_data.TABLE_KEY)` -#: = `customer_table_workspace__shared` -#: the GRANT is named `shares.field_oid(, key)` -#: = `customer_data:` -#: -#: `perm_scope.hidden_keys(user, module, fields)` hands its `module` argument straight through to -#: `field_grant_hidden`, which builds the oid from it. On a `ut_*` database the module argument IS -#: the table key, so W38-T16 never had to tell them apart; on a REGISTRY topic they differ, and a -#: door that claims the grant under the bucket name writes a record the wall will never look for. -#: MEASURED, not reasoned: the first run of this ticket's gate did exactly that, and user B was -#: refused a column that had been shared with them through the real share door, with a 200 at -#: every step ([[one-question-two-normalizers]]). -SHARE_TOPIC = MODULE - - -def _route_slug(label): - """A stable store key from a human label. Lower case, non-alphanumerics collapsed to `_`.""" - import re as _re - slug = _re.sub(r"[^a-z0-9]+", "_", str(label or "").strip().lower()).strip("_") - return f"{ROUTE_KEY_PREFIX}{slug[:48]}" if slug else "" - - -def _route_defs(session: Session): - """This topic's route-order columns MINUS the ones this session was not granted. - - ⛔ THE WALL IS THE SAME ONE THE GRID USES, NOT A SECOND OPINION. `hidden_keys` is where T16 - put the per-field grant check, so filtering on it here means the listing, the grid contract - and the row payload agree by construction. A column a reader cannot see must not appear here - either: the done-when says *does not see the field at all*, and a picker that names a column - whose values are withheld has already leaked its existence. - """ - import aios_grid - import core.perm_scope as perm_scope - - all_defs = shared_fields(st=session.runtime) or {} - defs = {k: v for k, v in all_defs.items() - if isinstance(v, dict) and v.get("kind") == ROUTE_KIND} - if not defs: - return {} - hide = perm_scope.hidden_keys( - session.user, MODULE, _merge_shared_fields(list(aios_grid.FIELDS), all_defs), - st=session.runtime) - return {k: v for k, v in defs.items() if k not in hide} - - -@router.get("/customers/route-order") -def route_order_list(session: Session = Depends(module_gate(MODULE))): - """The route-order columns this session may see, with the fingerprint each was solved from. - - ⭐⭐ THE FINGERPRINT IS WHY THIS ROUTE EXISTS RATHER THAN THE CLIENT READING `fields`. - Staleness is DERIVED, never stored: what is written down is the INPUT FINGERPRINT the numbers - were produced from, so *is this order still current* is a question asked at READ time against - what is on screen NOW, and can never itself be out of date. A stored `stale: true` is a fact - about a moment that has already passed. - """ - out = [] - for key, defn in sorted(_route_defs(session).items()): - route = defn.get("route") if isinstance(defn.get("route"), dict) else {} - out.append({ - "key": key, - "label": defn.get("label") or key, - "inputsHash": str(route.get("inputsHash") or ""), - "roundTrip": bool(route.get("roundTrip")), - "startPid": route.get("startPid"), - "stops": route.get("stops"), - "solvedAt": route.get("solvedAt") or "", - "solvedBy": defn.get("createdBy") or "", - # Who may re-solve it. The same creator-or-admin wall the write door enforces, said on - # the way out so the client can grey the control instead of discovering a 403. - "mine": bool(session.admin - or str(defn.get("createdBy") or "") == session.uname), - }) - return {"fields": out} - - -@router.post("/customers/route-order") -def route_order_write(body: dict = Body(default=None), - session: Session = Depends(module_gate(MODULE))): - """Create (or re-solve) a tenant-wide route-order column and fill it, in ONE call. - - `{label, field?, ranks: {"": }, inputsHash, roundTrip?, startPid?, stops?}` - - ⛔ THE DEFINITION IS WRITTEN FIRST AND THE GRANT CLAIMED SECOND, which is `patch_shared_cell`'s - order and it is deliberate: the window where a column is MARKED and UNCLAIMED fails CLOSED - (governed, nobody granted, so only an admin and the creator see it, and an admin can share - it). The other order would leave a grant record pointing at nothing. - - ⛔ RE-SOLVING IS CREATOR-OR-ADMIN, WHICH CREATING IS NOT. Writing these numbers changes what - every account in the workspace reads, at once, for the whole cohort. That is the wall - `routes_tables.delete_shared_field` already applies to the destructive half of this stratum, - and a new door does not get to inherit the loose half of an asymmetry somebody has flagged. - A permitted teammate READS the numbers; they do not silently re-plan somebody's day. - """ - from core import shared_overlay - import core.perm_scope as perm_scope - from routes_grid import MAX_BULK_ROWS - - body = body if isinstance(body, dict) else {} - label = " ".join(str(body.get("label") or "").split())[:120] - key = str(body.get("field") or "").strip() or _route_slug(label) - if not key: - raise err(400, "bad_request", "a name is required for the route order column") - if not key.startswith(ROUTE_KEY_PREFIX): - raise err(400, "bad_field_key", - f"a route order column's key starts with '{ROUTE_KEY_PREFIX}', so it cannot " - f"collide with a column this database already owns") - import aios_grid - if key in {f.get("key") for f in aios_grid.FIELDS}: - raise err(400, "field_key_taken", - "this database already has a column with that key") - - defs = shared_fields(st=session.runtime) or {} - existing = defs.get(key) if isinstance(defs.get(key), dict) else None - if existing is not None: - # ⛔⛔ A REFUSAL MUST NOT DESCRIBE A COLUMN THE CALLER CANNOT SEE. The two refusals below - # name the column's KIND and its CREATOR, which is exactly the information the field wall - # exists to withhold: a stranger who guesses the key would otherwise learn that a route - # order exists on this database and who planned it. `routes_shares._can_see_object` makes - # the same choice for the same reason (a non-grantee gets the answer a non-existent id - # gets). The name is already taken either way, so the honest refusal says only that. - if key in perm_scope.hidden_keys( - session.user, MODULE, - _merge_shared_fields(list(aios_grid.FIELDS), defs), st=session.runtime): - raise err(400, "field_key_taken", - "that column name is already in use on this database") - if existing.get("kind") != ROUTE_KIND: - raise err(400, "not_a_route_column", - "that column is shared but it is not a route order column, so re-solving " - "it would overwrite values this door did not write") - owner = str(existing.get("createdBy") or "") - if not session.admin and owner != session.uname: - raise err(403, "forbidden", - f"a route order can be re-solved by the person who planned it or by an " - f"administrator. This one was planned by {owner or 'somebody else'}, and " - f"re-solving it would change the visit numbers for every account at once") - - raw = body.get("ranks") - if not isinstance(raw, dict) or not raw: - raise err(400, "bad_ranks", 'expected {ranks: {"": }}') - # ⛔ REPORTED, NEVER TRUNCATED (standing rule 1's second sentence, and `MAX_BULK_ROWS`' own - # note). The ceiling is `routes_grid`'s so there is ONE of them, not two that drift. - if len(raw) > MAX_BULK_ROWS: - raise err(400, "too_many_rows", - f"at most {MAX_BULK_ROWS} records per route; this one carried {len(raw)}") - - pool = allowed_pids(session) - ranks, not_in_pool, bad_value = {}, [], [] - for raw_pid, value in raw.items(): - try: - pid = int(raw_pid) - except (TypeError, ValueError): - not_in_pool.append(str(raw_pid)[:40]) - continue - # ⚠ THE POOL IS THE WALL, and it is the SAME predicate the read path applies - # (`apply_row_scope` inside `allowed_pids`), so a caller cannot number a record they - # could not be shown, including one in the other business unit. - if pid not in pool: - not_in_pool.append(str(raw_pid)[:40]) - continue - # ⛔ A BOOL IS AN `int` IN PYTHON and `True` would store as a visit number 1. Excluded by - # name rather than by hoping nobody sends one. - if isinstance(value, bool) or not isinstance(value, int) or value < 1: - bad_value.append(str(raw_pid)[:40]) - continue - ranks[pid] = value - - # ⛔⛔ A PARTLY HONOURED ROUTE IS NOT A ROUTE, AND THIS REFUSES THE WHOLE CALL RATHER THAN - # WRITING THE PART IT COULD. Caught by this ticket's own gate: a body carrying one record - # outside the caller's book still produced a clean 1..N over what was left, so the door minted - # a permanent tenant-wide column and filled it with a SHORTER route than the one that was - # solved. Every number in it was plausible and the day was wrong. - # ⚠ REPORTED, WITH THE SAMPLE, which is standing rule 1's second sentence: the refusal names - # what it could not take, so the caller can fix it rather than guess. - if not_in_pool: - raise err(400, "rows_not_in_your_book", - f"{len(not_in_pool)} of those records are not in your book, so the route " - f"cannot be written as it was solved. First few: " - f"{', '.join(sorted(not_in_pool)[:5])}") - if bad_value: - raise err(400, "rows_not_a_visit_number", - f"a visit number is a whole number from 1 upwards; {len(bad_value)} records " - f"carried something else. First few: {', '.join(sorted(bad_value)[:5])}") - if not ranks: - raise err(400, "no_rows_in_your_book", - "none of those records are in your book, so there is nothing to number") - # ⛔ A ROUTE IS A SEQUENCE, SO THE NUMBERS ARE 1..N WITH NO REPEAT AND NO GAP. A body that - # arrives truncated, doubled or partly applied would otherwise land as a plausible half - # route: every row carrying a number, and the day in the wrong order. - seq = sorted(ranks.values()) - if seq != list(range(1, len(seq) + 1)): - raise err(400, "not_a_sequence", - f"a route order is the numbers 1 to {len(seq)}, each used once. This one " - f"carried {len(seq)} records numbered up to {seq[-1]} with " - f"{len(seq) - len(set(seq))} repeated") - - stamp = time.strftime("%Y-%m-%d %H:%M") - defn = { - "key": key, - "label": label or (existing or {}).get("label") or key, - # ⛔ `int`, NEVER `text`. A text column sorts 1, 10, 11, 2 — fully populated, entirely - # plausible, wrong, and named by no gate. `patch_shared_cell` defaults to text; this door - # cannot. - "type": "int", - "source": "overlay", - "shared": True, - "kind": ROUTE_KIND, - # ⭐⭐ THE PER-FIELD GRANT MARKER (T16). It is an EXPLICIT write-once declaration and it is - # what makes the wall fail CLOSED: a grant wall has the opposite absence-polarity to a - # deny wall, so keying visibility on "does a grant record exist" would publish this column - # to the whole tenant on one unreadable read of `object_shares`. Stamped, never inferred. - perm_scope.FIELD_GRANT_MARK: True, - "createdBy": (existing or {}).get("createdBy") or session.uname, - # ⭐ THE FINGERPRINT RIDES THE DEFINITION, ONCE. One `planRoute` run solves the whole - # cohort, so this string is identical for every record in it; per row it would be the same - # value written N times, and `shared_overlay._value` raises on a dict anyway. - "route": { - "inputsHash": str(body.get("inputsHash") or "")[:64], - "roundTrip": bool(body.get("roundTrip")), - "startPid": int(body["startPid"]) if isinstance(body.get("startPid"), int) - and not isinstance(body.get("startPid"), bool) else None, - "stops": len(ranks), - "solvedAt": stamp, - }, - } - shared_overlay.put_field(_shared_key(), key, defn, st=session.runtime) - if existing is None: - try: - import core.shares as shares - # ⚠ THE EMPTY ENTRY LIST IS THE POINT. `set_grants` keeps a record with an owner and - # no entries, so "shared with nobody" is STORED and is a different fact from "never - # shared". Without the owner the column is unmanageable: `may_administer` fails closed - # on an ownerless record, so nobody could ever share it. - shares.set_grants("field", shares.field_oid(SHARE_TOPIC, key), [], - owner=session.uname, st=session.runtime) - except Exception: # noqa: BLE001 - # The mark is already written, so a failed claim fails CLOSED: governed, nobody - # granted, admin-and-creator only. Recoverable. The other order is not. - pass - - # ⛔ AND THE RECORDS THAT LOST THEIR NUMBER ARE CLEARED. A re-solve over a SMALLER cohort - # would otherwise leave the previous run's ranks sitting on the records that dropped out — - # plausible integers, from a route nobody is driving. `""` is the house spelling of an empty - # overlay cell (`rows_from_pool` defaults an absent one to exactly that), so this needs no new - # vocabulary and no tombstone nobody else reads. - stale = {} - for raw_pid, cells in (shared_cells(pool, st=session.runtime) or {}).items(): - if not isinstance(cells, dict) or cells.get(key) in (None, ""): - continue - try: - gone = int(raw_pid) - except (TypeError, ValueError): - continue - if gone not in ranks: - stale[gone] = {key: ""} - written = shared_overlay.put_rows( - _shared_key(), {**{p: {key: v} for p, v in ranks.items()}, **stale}, - st=session.runtime) - - out = {"ok": True, "field": key, "label": defn["label"], "type": "int", - "stops": len(ranks), "cleared": len(stale), - "rows_written": len(written), "inputsHash": defn["route"]["inputsHash"], - "solvedAt": stamp} - return out +"""routes_customers.py — X2's read + write of the customer table, BU-SCOPED (EXIT-3b / EXIT-2a). + +Two things happen here that did not happen in the pre-wave `main.py`: + + 1. **ROWS ARE SCOPED TO THE SESSION.** The old `/api/customers` served `cl.pool()` — the whole + book, to anyone holding the shared APP_PASSWORD. Now the pool is built with + `(team_id, agent_name)` derived from the USER RECORD, so a Royal-only user never receives a + Fisch row and an agent-linked login never receives another rep's book. The scope is applied + at the QUERY, not as a post-filter, so there is no moment at which the other BU's rows exist + in this response. + + 2. **THE OVERLAY FORK IS GONE.** `aios-web/api/data/overlay.json` was a SECOND writable home + for the same user-owned fields the Streamlit app keeps in the tenant store — two truths, and + whichever process you asked last was right. Reads and writes now both go through + `modules.customer_data`'s table-workspace functions: ONE store (C1c, ARCHITECTURE §1a rule 3). + +⚠ ONE STORE IS NOT YET ONE CACHE (strangler-period, booked honestly). `core.store.get()` is +cache-first per PROCESS, so a write from the Streamlit container is invisible to a running API +container until its cache is refreshed, and vice versa. Deleting the fork removes the second +SOURCE OF TRUTH; it does not make the two runtimes coherent. The fix is X4/Postgres (task C-4, +owner-blocked on B-3), and no test here may claim read-your-writes ACROSS runtimes — a TestClient +proof is single-process and would report green on exactly the thing that is still broken. +""" +import time + +from fastapi import APIRouter, Body, Depends + +import scope_cache +from deps import Session, err, module_gate, perms + +router = APIRouter(prefix="/api/v1") + +#: The surface these routes serve. Both legs are gated on it, so a user without the grant gets a +#: 403 rather than an empty table that looks like "you have no customers". +MODULE = "customer_data" + +_CACHE_TTL = 900 # the pool build is slow (Odoo + reconciliation); 15 min, as before + + +def _pool_rows(session: Session): + """The reconciled Odoo pool for this session's SCOPE — the slow part, and the only part that + may be shared between users. + + ⛔ WHAT MAY BE CACHED HERE, AND WHY THE LINE IS EXACTLY HERE. The cache key is + `(team_id, agent)` and the cached value is the raw pool: Odoo-source columns only. That is + genuinely scope-shaped — two users with the same BU and the same book are asking the same + question, and `cl.pool()` is expensive (an Odoo pull plus reconciliation). + + The FULL PAYLOAD is NOT cacheable on this key, and caching it here was a real defect I shipped + and then removed. `fields` comes from `fields_from_workspace(ws)` — that user's own `custom_` + and `measure_` columns — and every overlay cell comes from `ws['overlays']`; the table + workspace is read as `data[username]` and written as `patch_table_overlay(uname, …)`, i.e. + PER USER. So two Royal-only users with no agent link share `(6, None)` and the second one + would have been served the FIRST one's private notes and private columns. The scope key was + right for the pool and wrong for everything wrapped around it. + + It survived a green 129-check battery because every fixture user had a DISTINCT + `(team_id, agent)` pair, so no two of them ever collided on the key — the test set could not + express the bug. `verify_api.py` now carries a same-scope second user for exactly this. + """ + rt = session.runtime + team_id, agent = _team_agent(session) + return _pool_for(rt, team_id, agent) + + +def _pool_for(rt, team_id, agent): + """The cached pool for an explicit scope — session-free so the prewarm thread and the + stale-refresh path can call it. STALE-WHILE-REFRESH (scope_cache): once a copy exists no + request blocks on the 10–30s Odoo rebuild again; only a scope's FIRST-ever build does. + + DEBT-2 (2026-08-04): while the tenant's RESOLVED Odoo source is PAUSED this path never + goes live — it serves the in-process copy at any age, else the persisted pause-time + snapshot (restart-safe), else answers 503. Never a WIDER scope's snapshot: handing a + scoped user the consolidated rows would widen their book, which is worse than an error.""" + import modules.customer_data as cl + import routes_keychain + + key = ("pool", team_id, agent) + + if routes_keychain.odoo_paused(rt): + hit = rt.pool_cache.get(key) + if hit: + return hit[1] + snap = routes_keychain.load_pool_snapshot(rt, team_id, agent) + if snap is not None: + rt.pool_cache[key] = snap # seed, so the memo stamps stay coherent + return snap[1] + raise err(503, "connector_paused", + "this data source is paused and no snapshot exists for your scope — " + "an admin can resume it under Settings → Connectors") + + def _build(): + # ⚠ THE SCOPE GOES INTO THE BUILDER. `pool(agent_name, team_id)` is the same reconciled + # builder the Streamlit page uses — passing the scope here is what makes the isolation a + # property of the QUERY instead of a filter someone can forget to apply downstream. + return cl.pool(agent, team_id) + + def _evict(): + # Bounded: a scope cache that only ever grows is a memory leak in a shared process. + if len(rt.pool_cache) > 16: + for stale in sorted(rt.pool_cache, key=lambda k: rt.pool_cache[k][0])[:8]: + rt.pool_cache.pop(stale, None) + + return scope_cache.get(rt.pool_cache, key, _CACHE_TTL, _build, _evict) + + +def warm_default(rt): + """Boot prewarm: the consolidated pool `(None, None)` — the scope every admin and every + all-BU account lands on. Called from main.py's prewarm thread only.""" + _pool_for(rt, None, None) + + +def _team_agent(session: Session): + """The `(team_id, agent)` this session's POOL is built with — ONE derivation point, used by + `_pool_rows` and `grid_assembly` alike, so the cache key and the query can never disagree. + + ⛔ WAVE 15 R1: THIS NOW COMES FROM THE PERMANENT FILTER (C-PERM amendment 3). `team_id` is + not a row filter — `customer_data._pool_build` passes it into `cust._cust_rev` three times + and into `_cadence_bulk`, so it decides what `rev`/`ly`/`ltm`/`aov`/`status` MEAN. Enforcing + a BU purely as a post-filter would keep the row list right and silently consolidate every + number. So the pushdown survives as a DERIVATION OF the declared filter rather than a second + wall beside it, and `perm_scope.derive_pool_scope` falls back to the legacy `bus`/`agent` + derivation for any record the migration has not reached yet. + """ + import core.perm_scope as perm_scope + return perm_scope.derive_pool_scope(session.user, MODULE) + + +def _pool_stamp(rt, team_id, agent): + """The cached pool's build timestamp — the DATA STAMP in every measure-memo key, so a pool + refresh invalidates the memoised answers exactly when the underlying rows changed.""" + entry = rt.pool_cache.get(("pool", team_id, agent)) + return entry[0] if isinstance(entry, tuple) and entry else 0 + + +def _measure_err(tag, e): + try: + import harness.telemetry as _tel + _tel.error(f"api:{tag}", e) + except Exception: + pass + + +# ══════════════════════════ THE TENANT-WIDE STRATUM, ON THE CUSTOMER TOPIC (W38-T20 / D-425) ══ +# +# ⛔⛔ WHY THIS FILE GREW A SHARED STRATUM AT ALL. `core/shared_overlay.py` has been generic since +# W29-T62, `routes_products.py` has merged it since W30-T36 and `routes_tables.py` since W38-T16 — +# and the CUSTOMER topic had neither a write door nor a read merge. Every user-created column here +# lives in `data[username]`, so two accounts looking at "the same" column are looking at two +# columns. That is fine for a private note and fatal for a ROUTE ORDER: a visit sequence one rep +# can see and their colleague cannot is not a plan, it is a rumour. +# +# ⛔ ONE SPELLING OF THE BUCKET. `modules.customer_data.TABLE_KEY` is the per-user workspace key +# and `shared_overlay.bucket()` derives `__shared` from it. Resolving it here rather than +# writing the string means the write door and the read merge cannot disagree about where the +# values live — which is exactly the failure T16 found on the materialised `ut_*` tables, where +# `patch_shared_cell` wrote into a bucket no reader ever opened. + + +def _shared_key(): + """The store key this topic's per-user AND tenant-wide strata are both named from.""" + import modules.customer_data as cl + return cl.TABLE_KEY + + +def shared_fields(st=None): + """`{field_key: Field}` — the columns this topic shares tenant-wide. + + Unscoped on purpose, exactly as `shared_overlay.fields` is: a shared column's EXISTENCE is + tenant-wide by definition. WHO MAY SEE IT is a separate question, answered one layer up by + `perm_scope.hidden_keys` (the per-field grant wall T16 landed), and WHOSE ROWS by `cells`. + """ + from core import shared_overlay + try: + return shared_overlay.fields(_shared_key(), st=st) + except Exception: # noqa: BLE001 + # Lenient like every other display read: an unreachable store degrades to "nothing is + # shared yet", never to a 500 on a grid that would otherwise render. The WALL does not + # degrade with it — `field_grant_hidden` hides a marked column it cannot resolve. + return {} + + +def shared_cells(pids, st=None): + """`{"": {key: value}}` for the rows named by `pids`, and ONLY those. + + ⛔ `pids` IS THE ROW WALL, PASSED AND NEVER DEFAULTED. `shared_overlay.cells` refuses an + "everything" read by signature for this reason; the set handed in is the one `grid_assembly` + has already narrowed with `apply_row_scope`, so a cell belonging to the other BU has nothing + to attach itself to. + """ + from core import shared_overlay + try: + return shared_overlay.cells(_shared_key(), list(pids or ()), st=st) + except Exception: # noqa: BLE001 + return {} + + +def _merge_shared_fields(fields, defs): + """`fields` PLUS the tenant-wide columns this topic declares — `routes_tables._ut_shared_fields` + on the customer topic. + + ⚠ MERGED BEFORE THE WALL, NEVER AFTER. `hidden_keys` is a TRANSITIVE closure, so it must run + on the WHOLE contract: a formula over a shared column that reads a hidden one sits outside the + closure's reach otherwise and carries the hidden value out wearing a second name. It is also + the only order in which `field_grant_hidden` can ever see the `granted` marker at all — merge + afterwards and the per-field wall is inert while every test still passes. + ⚠ A key the canonical contract already declares WINS. A shared column is an ADDITION to this + database's contract, never a redefinition of a column it already has. + """ + if not defs: + return fields + have = {f.get("key") for f in (fields or ()) if isinstance(f, dict)} + return list(fields or ()) + [dict(f, source="overlay") + for k, f in defs.items() if k not in have] + + +def grid_assembly(session: Session, scope: str = "customer", storage_key: str = "", + consume_corrections: bool = True): + """ONE assembly of this session's grid state, shared by `/customers`, `/workspace` and the + events route (2026-07-31 — the standalone measure gap, owner item 1). + + What it adds over the pre-wave hand-rolled `_payload` loop, and why it replaced it: + + * rows go through `aios_grid.rows_from_pool` — the SAME builder the embedded host uses, + so `lat`/`lon` (the Map view's data), `_created` and every derived column ride each row + by construction instead of by a second loop that drifts. The hand loop was written to + mirror the pre-Map contract and silently dropped the coordinates: the shell's Map view + had nothing to plot ("the Map no longer works"). + * `derived` carries the cohort column's cells AND the measure columns' values, resolved + through `core.measure_resolve` (EXIT-5's extraction of the `_cl_measure_*` family). + Without them every measure column the owner built rendered BLANK in the shell. + * `measures` (the offer) and `measure_sets` (condition answers) are computed here so the + events route can finally validate measure fields/conditions instead of refusing them + (an empty `measure_offer` made `clean_measure_field` reject every create over HTTP). + + Memos live on the TENANT RUNTIME (`rt.measure_memo` / `rt.mset_memo`) — bounded by the + module's own clear-past-cap rule, keyed on (stamp, scope, pool identity, question), nothing + user-shaped in them. + """ + import aios_grid + from core import grid_events, measure_resolve + + import core.perm_scope as perm_scope + + rt = session.runtime + team_id, agent = _team_agent(session) + rows_src = _pool_for(rt, team_id, agent) + # ⛔ THE ROW WALL, APPLIED BEFORE `pids` IS TAKEN. Everything downstream is bounded by that + # frozenset — `allowed_pids` for the workspace, cohort membership, measure resolution — so + # scoping here means a row this account may not see never enters ANY of them, rather than + # being filtered out of one payload and surviving in another. + # + # Evaluated against the CANONICAL field list, not the per-user assembled one, for two + # reasons: the assembled list is not built yet (it needs `pids`), and a permanent filter may + # only ever name a canonical field anyway — `routes_admin._clean_perms` validates it against + # exactly this schema and 400s otherwise. `permits()` denies on anything it cannot answer. + rows_src = perm_scope.apply_row_scope(rows_src, session.user, MODULE, aios_grid.FIELDS) + pids = frozenset(r["pid"] for r in rows_src if r.get("pid") is not None) + # ⭐ W38-T20 — THE COLUMN DEFINITIONS ARE READ **ONCE** PER ASSEMBLY AND THREADED, because + # `core.store.get` deep-copies whatever it hands back on every call. Three consumers want this + # dict on one request (the write ctx's wall, the read merge, and the closure), and letting each + # take its own copy is the shape D-214 spent a whole ticket removing one document over. + _defs = shared_fields(st=rt) + ws = grid_events.table_workspace( + _ctx_for(session, pids, defs=_defs), allowed_pids=pids, + consume_corrections=consume_corrections) + # ⭐⭐ W38-T20 / D-425 — THE TENANT-WIDE CELLS, LAYERED OVER THE PER-USER ONES, IN THE + # ASSEMBLY SO EVERY CONSUMER SEES ONE TRUTH. `routes_grid`'s /workspace route serves + # `workspace["overlays"] = g["ws"].get("overlays")` verbatim and `_payload` hands the same + # dict to `rows_from_pool`, so merging HERE reaches both without touching either file. + # + # ⚠ SAFE TO MUTATE, and checked rather than assumed (the same check `product_assembly` + # records): `table_workspace` reads through `store.get`, which deep-copies, so `ws` is a + # detached copy and nothing writes it back. A shared value can never leak INTO the per-user + # bucket by way of this merge. + # ⚠ SHARED WINS PER KEY. The whole point of the stratum is that every reader sees the same + # number, so a per-user leftover under the same key is stale by construction. It is also what + # makes D-423 recoverable rather than permanent: a pre-fix per-user edit is shadowed, not + # promoted. + _shared = shared_cells(pids, st=rt) + if _shared: + _ov = dict(ws.get("overlays") or {}) + for _pid, _cells in _shared.items(): + _ov[_pid] = {**(_ov.get(_pid) or {}), **_cells} + ws["overlays"] = _ov + workspace, fields, views, lists = aios_grid.workspace_wire( + ws, session.uname, set(pids), defs={}, scope_key=scope, storage_key=storage_key) + # ⭐⭐ W38-T20 — AND THE COLUMN DEFINITIONS, BEFORE THE WALL. See `_merge_shared_fields`: this + # position is load-bearing twice, once for the transitive closure and once because it is the + # only order in which the per-field grant marker is ever presented to `hidden_keys`. + fields = _merge_shared_fields(fields, _defs) + # THE FIELD WALL — a TRANSITIVE closure (C-PERM amendment 5), so hiding a field also hides + # every formula computed FROM it. Formulas evaluate in the browser from `{ref}`s, so + # shipping a dependent formula while withholding its input either leaks the input through + # the formula's value or silently computes a wrong one; only removing both is coherent. + # Applied AFTER workspace_wire because custom + measure columns are what it must cover. + # ⭐ W38-T20 — `st=rt` IS NOT TIDINESS. `perm_scope.field_grant_hidden` resolves a marked + # column against `object_shares`, and without a tenant handle it reads the module-default + # bucket: on any tenant but #0 that finds no grant, and no grant on a MARKED column means + # HIDDEN. So an unthreaded `st` UNDER-shares (a grantee cannot see their own column) rather + # than over-shares — visible and reportable, but still wrong, and `visible_fields`' own note + # requires the two calls to agree about it. + hidden = perm_scope.hidden_keys(session.user, MODULE, fields, st=rt) + if hidden: + fields = [f for f in fields if f.get("key") not in hidden] + # The field LIST and the ROW payload are two different wires. Narrowing only the first + # would leave the value sitting in the second, where anything can read it. + rows_src = [perm_scope.strip_row(r, hidden) for r in rows_src] + # ⛔⛔ AND THE OVERLAY DICT IS A THIRD WIRE, WHICH IS NEW THIS TICKET AND WAS A HOLE + # BEFORE IT. `rows_from_pool` iterates `fields`, so a narrowed contract already keeps a + # hidden key off a ROW — but `/workspace` serves `ws["overlays"]` RAW, and that dict is + # where both the per-user cells and (as of the merge above) the tenant-wide ones sit. + # One narrowing cannot speak for a wire it never touches; `routes_tables` writes the same + # sentence over its own `shared_cells`, and `routes_odoo_tables` over its `overlays`. + # ⚠ `ws` AND NOT `workspace`: `routes_grid.workspace` copies the dict ACROSS + # (`workspace["overlays"] = g["ws"].get("overlays")`) after this returns, so narrowing the + # source is what reaches the wire. Narrowing the copy would be narrowing a key that gets + # overwritten a moment later. + _ov = ws.get("overlays") or {} + if _ov: + ws["overlays"] = {pid: {k: v for k, v in (cells or {}).items() if k not in hidden} + for pid, cells in _ov.items()} + + today = time.strftime("%Y-%m-%d") + stamp = _pool_stamp(rt, team_id, agent) + # ⭐⭐ W38-T19 — THE METRICS CAPABILITY, AND THIS GRAIN NEEDS **THREE** GUARDS WHERE THE + # OTHER TWO NEED ONE. `routes_products` and `routes_tables` funnel their cells and their + # condition answers through helpers that short-circuit on an empty `offer`, so emptying the + # offer there stops the whole feature. `core.measure_resolve` does not take an offer at all: + # `condition_sets` and `column_values` both re-derive their work from the caller's OWN saved + # views and field list. So gating only the offer here would take the Metric kind off the + # picker and refuse new creates while an EXISTING Metric column kept computing and an + # EXISTING measure condition kept resolving — a revoked capability still answering, on the + # grain with the most of them. Three calls, one predicate. + may_metrics = perm_scope.may_metrics(session.user, MODULE) + measures = measure_resolve.offer(team_id, on_error=_measure_err) if may_metrics else [] + measure_sets = measure_resolve.condition_sets( + [v.get("config") or {} for v in (views or [])], None, team_id, pids, today, stamp, + rt.mset_memo, on_error=_measure_err) if may_metrics else {} + # The derived channel: cohort membership cells + measure column values, ONE dict — the + # same read-only channel the embed host hands to rows_from_pool. + derived = aios_grid.cohort_cells(lists) + # ⭐ W33-T43 / owner item 12 ("One unique ID per database always"). The customer grid carried + # NO Odoo id column at all, while its retiring twin `ut_odoo_customers` carried `partner_id` + # as its join key — so the merge would have lost the one value every Odoo document joins on. + # + # ⛔ IT IS DERIVED, NOT A POOL COLUMN, AND THAT IS THE WHOLE POINT: a customer row's `pid` IS + # the `res.partner` id (`modules/customer_data.pool` mints it that way and the identity is + # asserted against Odoo in that module). Adding it to the pool would be a SECOND source for + # one fact, which is the class of defect item 12 is about. This channel exists for exactly + # this — a value the host knows per render and the pool has no business storing. + for pid in pids: + derived.setdefault(pid, {})["partner_id"] = pid + # ⭐⭐ W38-T19 — SKIPPED ENTIRELY WHEN THE CAPABILITY IS REVOKED, rather than filtered after. + # `column_values` selects its own subjects (`isinstance(f.get('measure'), dict)`) off the + # field list, so there is no argument that could narrow it; not calling it is the narrowing. + # ⚠ THE COLUMN STAYS AND ITS CELLS GO BLANK, which is this channel's OWN documented degrade + # ("blank is could not compute, 0 is a real zero") and is what the product and user-table + # grains already do under an empty offer. Deleting the column instead would be a second, + # louder behaviour for the same fact on one surface out of three, and it would destroy a + # definition the admin can restore with one tick. + for pid, cells in (measure_resolve.column_values( + fields, team_id, pids, today, stamp, rt.measure_memo, + on_error=_measure_err) if may_metrics else {}).items(): + derived.setdefault(pid, {}).update(cells) + + return {"rows_src": rows_src, "pids": pids, "ws": ws, "workspace": workspace, + "fields": fields, "views": views, "lists": lists, "derived": derived, + "measures": measures, "measure_sets": measure_sets, "today": today, + "team_id": team_id} + + +def _payload(session: Session): + """`{fields, rows, today, docs, pulled_at}` — X2's shape, which `verify_fields_contract.py` + referees. Rows are now built by `aios_grid.rows_from_pool` (embed == standalone by + construction); see `grid_assembly` for what that fixed. + + ⭐ `docs` joined the shape in wave 30 (W30-T37 / contract C4). Named here rather than left to + the reader because a docstring that still lists the OLD shape is a stale comment on correct + code — this repo's D-73 — and it is the first thing anyone greps to learn the payload. + + ⚠ `rows_src` is the SHARED cached list — `rows_from_pool` reads it and builds NEW dicts, + never mutating a cached row (the same-scope-second-user leak rule). + """ + import aios_grid + + g = grid_assembly(session) + rows = aios_grid.rows_from_pool( + g["rows_src"], g["fields"], g["ws"].get("overlays"), derived=g["derived"]) + # ⭐ C4 / D-138 (W30-T37) — THE DOCUMENTS PRODUCER FOR THE CUSTOMER SCOPE. The write door + # (`doc_add`/`doc_fetch`/`doc_delete`) never stopped working and every client half is + # complete; what vanished with `app.py` at EXIT-6 was the only thing that ever set this key. + # All six `onDoc*` handlers in `CustomerGrid.tsx` read `payload?.docs ? … : undefined`, so an + # ABSENT key — not a broken one — is what has been switching the whole feature off. + # + # ⛔ IMPORTED, NEVER RE-SERIALISED. `core.grid_events.docs_for` is the ONE serialiser and + # `routes_tables` (the `ut_*` scope) calls the SAME function with the same argument order. + # A matching pair here is precisely how the wave-29 close-out reintroduced its own defect in + # the opposite direction inside a single commit ([[one-question-two-normalizers]]). + # ⚠ `g["pids"]` is the row set this session is ALREADY scoped to — `docs_for` has no + # "every document in the tenant" mode to reach for, deliberately. + from core import grid_events as _ge + return {"fields": g["fields"], "rows": rows, + # `today` rides the payload because every relative date condition must resolve against + # the TENANT's day, never the browser's — a client that falls back to its own clock + # disagrees with the server for everyone west of it. + "today": g["today"], + "docs": _ge.docs_for(g["pids"], scope_key="customer", uname=session.uname, + admin=session.admin, st=session.runtime), + "pulled_at": time.strftime("%Y-%m-%d %H:%M")} + + +def _ctx_for(session: Session, pids, defs=None): + """An EventCtx for the READ path — no fallback workspace, so a store outage is a 503 rather + than a phantom in-memory workspace an API request cannot persist. + + `defs` (W38-T20) is this topic's tenant-wide column definitions when the caller already holds + them; absent, they are read. One read per assembly rather than one per question asked of it. + """ + from core import grid_events + return grid_events.EventCtx( + uname=session.uname, allowed_pids=frozenset(pids or ()), fields=[], + # C-PERM: the write wall's field half. Computed from the CANONICAL contract PLUS the + # tenant-wide columns, because `fields=[]` here — the closure only needs the schema, not + # this user's column list, and a runtime column is part of that schema now. + hidden_keys=_hidden_for(session, defs=defs), + admin=session.admin, fallback_ws=None, seen_ids={}) + + + +def _hidden_for(session: Session, defs=None): + """The fields this session's permissions hide — the write wall's half of C-PERM. + + Read paths strip these from both wires so they cannot be SEEN; this is what stops them + being WRITTEN by a caller who knows the key. Evaluated against the canonical contract, the + same schema `routes_admin._clean_perms` validates a hiddenFields entry against. + + ⭐⭐ W38-T20 — AND THE TENANT-WIDE COLUMNS ARE PART OF THAT CONTRACT NOW, WHICH IS A WALL AND + NOT A COMPLETENESS TIDY-UP. `patch_customer` builds its ctx with `fields=payload["fields"]`, + which carries the merged shared columns, so `grid_events.handle_one` would happily accept an + `overlay_patch` naming one. The canonical list cannot mention them (they are created at + runtime), so a wall computed from `aios_grid.FIELDS` alone answers "not hidden" for every + grant-governed column and the write door is open to a reader who was never granted it. + ⚠ Widening the field list can only ever ADD to the hidden set, never remove from it: the + closure hides what it is told to hide plus whatever depends on it. + """ + import aios_grid + import core.perm_scope as perm_scope + + if defs is None: + defs = shared_fields(st=session.runtime) + return perm_scope.hidden_keys( + session.user, MODULE, _merge_shared_fields(list(aios_grid.FIELDS), defs), + st=session.runtime) + +def allowed_pids(session: Session): + """The pids this session may touch — the POOL's own ids, so the write wall and the read scope + can never disagree. + + Reads `_pool_rows` rather than `_payload`: the wall only needs identities, and going through + the full payload would pay for a workspace read and a row assembly on every write. + + ⛔ THE PERMANENT FILTER APPLIES HERE TOO, AND FORGETTING IT IS A WRITE-WITHOUT-READ HOLE. + `_pool_rows` is built with the DERIVED pushdown, which expresses only what a `(team_id, + agent)` pair can express. Any part of the wall the pushdown cannot carry — `revenue > 1000`, + a nested group, a condition on any other column — leaves the pool WIDER than the filter. Read + paths close that gap with `apply_row_scope`; without the same call here the write wall would + be the wider set, and a restricted user could PATCH a row this API will not show them. + Same function, same order as `grid_assembly`, so the two walls cannot drift. + """ + import core.perm_scope as perm_scope + import aios_grid + + rows = perm_scope.apply_row_scope(_pool_rows(session), session.user, MODULE, + aios_grid.FIELDS) + return frozenset(r["pid"] for r in rows if r.get("pid") is not None) + + +@router.get("/customers") +def customers(session: Session = Depends(module_gate(MODULE))): + return _payload(session) + + +@router.patch("/customers/{pid}") +def patch_customer(pid: int, body: dict = Body(default=None), + session: Session = Depends(module_gate(MODULE))): + """Write the EDITABLE overlay stratum only — Odoo stays read-only, forever. + + Routed through `core.grid_events.handle_one` as an `overlay_patch` event rather than writing + the store directly: that handler is where the per-key `permissions.edit` wall, the pid wall + and the truncation rules live, and a second implementation of those would be a second set of + them to keep in step. The response reports what was ACCEPTED, which is not always what was + asked for. + """ + from core import grid_events + + updates = dict(body or {}) + if not updates: + raise err(400, "empty_patch", "no fields to update") + pool = allowed_pids(session) + if pid not in pool: + # 403, not 404: the pid may well exist — it is simply not in this session's book, and + # saying "no such customer" would confirm the opposite to anyone who guessed right. + raise err(403, "out_of_scope", "that customer is not in your book") + payload = _payload(session) + ctx = grid_events.EventCtx( + uname=session.uname, allowed_pids=pool, fields=payload["fields"], + admin=session.admin, fallback_ws=None, seen_ids={}, + hidden_keys=_hidden_for(session)) + try: + grid_events.handle_one( + {"id": f"patch:{pid}:{time.time_ns()}", "type": "overlay_patch", + "pid": pid, "updates": updates}, ctx) + except grid_events.StoreUnavailable: + raise err(503, "store_unavailable", + "the tenant store is unavailable — your change was not saved") + + # What actually landed, read back from the store rather than echoed from the request: a + # refused key or a truncated value must not be reported as accepted. + stored = (grid_events.table_workspace(_ctx_for(session, pool), allowed_pids=None) + .get("overlays") or {}).get(str(pid)) or {} + accepted = {k: stored.get(k) for k in updates if k in stored} + refused = sorted(k for k in updates if k not in accepted or stored.get(k) != str(updates[k])) + # No cache to patch: the overlay stratum is re-read from the store on every `_payload`, so + # read-your-writes within this runtime is a property of the design rather than of a + # write-through step somebody has to remember. (It was a write-through step while the whole + # payload was cached on a scope key — the arrangement that leaked one user's notes to + # another. See `_pool_rows`.) Cross-RUNTIME coherence is still not claimed: the module + # docstring says why, and Postgres is the fix. + out = {"ok": True, "pid": pid, "updates": accepted} + if refused: + out["refused"] = refused + return out + + +# ══════════════════════════════════ THE ROUTE-ORDER COLUMN (W38-T20 / ruling R7 / contract C1) ══ +# +# ⛔⛔ WHY THIS IS A DOOR HERE AND NOT A KIND IN THE COLUMN MENU. `ColumnMenu.onCreate` is the only +# persistence the menu has, and it lands a PER-USER `custom_` column through +# `aios_grid._field_extras`, which is a strict allowlist: a route-order bag created that way is +# silently stripped and its values are private to their author. That is the wave-19 `image` +# failure verbatim ("created, named, configured, gone", recorded in `_clean_geocode`'s own +# docstring) plus a done-when clause that cannot hold, because a colleague reading a per-user +# stratum gets a clean 200 with nothing in it. The `geocode` pseudo-kind is the precedent for the +# PICKER; its persistence half does not transfer. +# +# ⭐ SO THE COLUMN IS BORN SHARED. `shared_overlay.put_field` stores the definition verbatim (no +# allowlist), which is what lets the input fingerprint ride the DEFINITION rather than the rows — +# `shared_overlay._value` RAISES on a dict, so `{order, inputsHash}` could never be one cell, and +# one solve fingerprints the whole cohort identically anyway, so per-row would be the same string +# written N times. +# +# ⭐ AND THE NUMBER ON THE RECORD IS THE INVERSE OF THE PLANNER'S ANSWER. `mapProjection.planRoute` +# returns `order`, where `order[i]` is WHICH STOP is visited i-th; the cell holds the RANK. The +# client inverts it with `routeRanks` (gated in `map.test.ts` at the desktop shape, over a fixture +# chosen so the two differ); this door then refuses anything that is not a clean 1..N, so a +# truncated or double-posted body cannot land as a half-route. + +#: The key prefix every route-order column wears. `custom_` and `measure_` are the two existing +#: created-column namespaces and both are PER USER; this one is tenant-wide, so it takes its own +#: rather than borrowing a prefix whose readers assume a per-user home. +ROUTE_KEY_PREFIX = "route_" + +#: The marker on the stored definition that says WHAT this column is. Read by the GET below and by +#: the client; never inferred from the key, because a prefix is a naming convention and a +#: convention is not a declaration. +ROUTE_KIND = "route_order" + +#: ⛔⛔ THE TOPIC A FIELD GRANT IS NAMED UNDER, AND IT IS **NOT** THE STORE BUCKET. Two namespaces +#: meet on this column and they are spelled differently: +#: +#: the STRATUM lives at `shared_overlay.bucket(customer_data.TABLE_KEY)` +#: = `customer_table_workspace__shared` +#: the GRANT is named `shares.field_oid(, key)` +#: = `customer_data:` +#: +#: `perm_scope.hidden_keys(user, module, fields)` hands its `module` argument straight through to +#: `field_grant_hidden`, which builds the oid from it. On a `ut_*` database the module argument IS +#: the table key, so W38-T16 never had to tell them apart; on a REGISTRY topic they differ, and a +#: door that claims the grant under the bucket name writes a record the wall will never look for. +#: MEASURED, not reasoned: the first run of this ticket's gate did exactly that, and user B was +#: refused a column that had been shared with them through the real share door, with a 200 at +#: every step ([[one-question-two-normalizers]]). +SHARE_TOPIC = MODULE + + +def _route_slug(label): + """A stable store key from a human label. Lower case, non-alphanumerics collapsed to `_`.""" + import re as _re + slug = _re.sub(r"[^a-z0-9]+", "_", str(label or "").strip().lower()).strip("_") + return f"{ROUTE_KEY_PREFIX}{slug[:48]}" if slug else "" + + +def _route_defs(session: Session): + """This topic's route-order columns MINUS the ones this session was not granted. + + ⛔ THE WALL IS THE SAME ONE THE GRID USES, NOT A SECOND OPINION. `hidden_keys` is where T16 + put the per-field grant check, so filtering on it here means the listing, the grid contract + and the row payload agree by construction. A column a reader cannot see must not appear here + either: the done-when says *does not see the field at all*, and a picker that names a column + whose values are withheld has already leaked its existence. + """ + import aios_grid + import core.perm_scope as perm_scope + + all_defs = shared_fields(st=session.runtime) or {} + defs = {k: v for k, v in all_defs.items() + if isinstance(v, dict) and v.get("kind") == ROUTE_KIND} + if not defs: + return {} + hide = perm_scope.hidden_keys( + session.user, MODULE, _merge_shared_fields(list(aios_grid.FIELDS), all_defs), + st=session.runtime) + return {k: v for k, v in defs.items() if k not in hide} + + +@router.get("/customers/route-order") +def route_order_list(session: Session = Depends(module_gate(MODULE))): + """The route-order columns this session may see, with the fingerprint each was solved from. + + ⭐⭐ THE FINGERPRINT IS WHY THIS ROUTE EXISTS RATHER THAN THE CLIENT READING `fields`. + Staleness is DERIVED, never stored: what is written down is the INPUT FINGERPRINT the numbers + were produced from, so *is this order still current* is a question asked at READ time against + what is on screen NOW, and can never itself be out of date. A stored `stale: true` is a fact + about a moment that has already passed. + """ + out = [] + for key, defn in sorted(_route_defs(session).items()): + route = defn.get("route") if isinstance(defn.get("route"), dict) else {} + out.append({ + "key": key, + "label": defn.get("label") or key, + "inputsHash": str(route.get("inputsHash") or ""), + "roundTrip": bool(route.get("roundTrip")), + "startPid": route.get("startPid"), + "stops": route.get("stops"), + "solvedAt": route.get("solvedAt") or "", + "solvedBy": defn.get("createdBy") or "", + # Who may re-solve it. The same creator-or-admin wall the write door enforces, said on + # the way out so the client can grey the control instead of discovering a 403. + "mine": bool(session.admin + or str(defn.get("createdBy") or "") == session.uname), + }) + return {"fields": out} + + +@router.post("/customers/route-order") +def route_order_write(body: dict = Body(default=None), + session: Session = Depends(module_gate(MODULE))): + """Create (or re-solve) a tenant-wide route-order column and fill it, in ONE call. + + `{label, field?, ranks: {"": }, inputsHash, roundTrip?, startPid?, stops?}` + + ⛔ THE DEFINITION IS WRITTEN FIRST AND THE GRANT CLAIMED SECOND, which is `patch_shared_cell`'s + order and it is deliberate: the window where a column is MARKED and UNCLAIMED fails CLOSED + (governed, nobody granted, so only an admin and the creator see it, and an admin can share + it). The other order would leave a grant record pointing at nothing. + + ⛔ RE-SOLVING IS CREATOR-OR-ADMIN, WHICH CREATING IS NOT. Writing these numbers changes what + every account in the workspace reads, at once, for the whole cohort. That is the wall + `routes_tables.delete_shared_field` already applies to the destructive half of this stratum, + and a new door does not get to inherit the loose half of an asymmetry somebody has flagged. + A permitted teammate READS the numbers; they do not silently re-plan somebody's day. + """ + from core import shared_overlay + import core.perm_scope as perm_scope + from routes_grid import MAX_BULK_ROWS + + body = body if isinstance(body, dict) else {} + label = " ".join(str(body.get("label") or "").split())[:120] + key = str(body.get("field") or "").strip() or _route_slug(label) + if not key: + raise err(400, "bad_request", "a name is required for the route order column") + if not key.startswith(ROUTE_KEY_PREFIX): + raise err(400, "bad_field_key", + f"a route order column's key starts with '{ROUTE_KEY_PREFIX}', so it cannot " + f"collide with a column this database already owns") + import aios_grid + if key in {f.get("key") for f in aios_grid.FIELDS}: + raise err(400, "field_key_taken", + "this database already has a column with that key") + + defs = shared_fields(st=session.runtime) or {} + existing = defs.get(key) if isinstance(defs.get(key), dict) else None + if existing is not None: + # ⛔⛔ A REFUSAL MUST NOT DESCRIBE A COLUMN THE CALLER CANNOT SEE. The two refusals below + # name the column's KIND and its CREATOR, which is exactly the information the field wall + # exists to withhold: a stranger who guesses the key would otherwise learn that a route + # order exists on this database and who planned it. `routes_shares._can_see_object` makes + # the same choice for the same reason (a non-grantee gets the answer a non-existent id + # gets). The name is already taken either way, so the honest refusal says only that. + if key in perm_scope.hidden_keys( + session.user, MODULE, + _merge_shared_fields(list(aios_grid.FIELDS), defs), st=session.runtime): + raise err(400, "field_key_taken", + "that column name is already in use on this database") + if existing.get("kind") != ROUTE_KIND: + raise err(400, "not_a_route_column", + "that column is shared but it is not a route order column, so re-solving " + "it would overwrite values this door did not write") + owner = str(existing.get("createdBy") or "") + if not session.admin and owner != session.uname: + raise err(403, "forbidden", + f"a route order can be re-solved by the person who planned it or by an " + f"administrator. This one was planned by {owner or 'somebody else'}, and " + f"re-solving it would change the visit numbers for every account at once") + + raw = body.get("ranks") + if not isinstance(raw, dict) or not raw: + raise err(400, "bad_ranks", 'expected {ranks: {"": }}') + # ⛔ REPORTED, NEVER TRUNCATED (standing rule 1's second sentence, and `MAX_BULK_ROWS`' own + # note). The ceiling is `routes_grid`'s so there is ONE of them, not two that drift. + if len(raw) > MAX_BULK_ROWS: + raise err(400, "too_many_rows", + f"at most {MAX_BULK_ROWS} records per route; this one carried {len(raw)}") + + pool = allowed_pids(session) + ranks, not_in_pool, bad_value = {}, [], [] + for raw_pid, value in raw.items(): + try: + pid = int(raw_pid) + except (TypeError, ValueError): + not_in_pool.append(str(raw_pid)[:40]) + continue + # ⚠ THE POOL IS THE WALL, and it is the SAME predicate the read path applies + # (`apply_row_scope` inside `allowed_pids`), so a caller cannot number a record they + # could not be shown, including one in the other business unit. + if pid not in pool: + not_in_pool.append(str(raw_pid)[:40]) + continue + # ⛔ A BOOL IS AN `int` IN PYTHON and `True` would store as a visit number 1. Excluded by + # name rather than by hoping nobody sends one. + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + bad_value.append(str(raw_pid)[:40]) + continue + ranks[pid] = value + + # ⛔⛔ A PARTLY HONOURED ROUTE IS NOT A ROUTE, AND THIS REFUSES THE WHOLE CALL RATHER THAN + # WRITING THE PART IT COULD. Caught by this ticket's own gate: a body carrying one record + # outside the caller's book still produced a clean 1..N over what was left, so the door minted + # a permanent tenant-wide column and filled it with a SHORTER route than the one that was + # solved. Every number in it was plausible and the day was wrong. + # ⚠ REPORTED, WITH THE SAMPLE, which is standing rule 1's second sentence: the refusal names + # what it could not take, so the caller can fix it rather than guess. + if not_in_pool: + raise err(400, "rows_not_in_your_book", + f"{len(not_in_pool)} of those records are not in your book, so the route " + f"cannot be written as it was solved. First few: " + f"{', '.join(sorted(not_in_pool)[:5])}") + if bad_value: + raise err(400, "rows_not_a_visit_number", + f"a visit number is a whole number from 1 upwards; {len(bad_value)} records " + f"carried something else. First few: {', '.join(sorted(bad_value)[:5])}") + if not ranks: + raise err(400, "no_rows_in_your_book", + "none of those records are in your book, so there is nothing to number") + # ⛔ A ROUTE IS A SEQUENCE, SO THE NUMBERS ARE 1..N WITH NO REPEAT AND NO GAP. A body that + # arrives truncated, doubled or partly applied would otherwise land as a plausible half + # route: every row carrying a number, and the day in the wrong order. + seq = sorted(ranks.values()) + if seq != list(range(1, len(seq) + 1)): + raise err(400, "not_a_sequence", + f"a route order is the numbers 1 to {len(seq)}, each used once. This one " + f"carried {len(seq)} records numbered up to {seq[-1]} with " + f"{len(seq) - len(set(seq))} repeated") + + stamp = time.strftime("%Y-%m-%d %H:%M") + defn = { + "key": key, + "label": label or (existing or {}).get("label") or key, + # ⛔ `int`, NEVER `text`. A text column sorts 1, 10, 11, 2 — fully populated, entirely + # plausible, wrong, and named by no gate. `patch_shared_cell` defaults to text; this door + # cannot. + "type": "int", + "source": "overlay", + "shared": True, + "kind": ROUTE_KIND, + # ⭐⭐ THE PER-FIELD GRANT MARKER (T16). It is an EXPLICIT write-once declaration and it is + # what makes the wall fail CLOSED: a grant wall has the opposite absence-polarity to a + # deny wall, so keying visibility on "does a grant record exist" would publish this column + # to the whole tenant on one unreadable read of `object_shares`. Stamped, never inferred. + perm_scope.FIELD_GRANT_MARK: True, + "createdBy": (existing or {}).get("createdBy") or session.uname, + # ⭐ THE FINGERPRINT RIDES THE DEFINITION, ONCE. One `planRoute` run solves the whole + # cohort, so this string is identical for every record in it; per row it would be the same + # value written N times, and `shared_overlay._value` raises on a dict anyway. + "route": { + "inputsHash": str(body.get("inputsHash") or "")[:64], + "roundTrip": bool(body.get("roundTrip")), + "startPid": int(body["startPid"]) if isinstance(body.get("startPid"), int) + and not isinstance(body.get("startPid"), bool) else None, + "stops": len(ranks), + "solvedAt": stamp, + }, + } + shared_overlay.put_field(_shared_key(), key, defn, st=session.runtime) + if existing is None: + try: + import core.shares as shares + # ⚠ THE EMPTY ENTRY LIST IS THE POINT. `set_grants` keeps a record with an owner and + # no entries, so "shared with nobody" is STORED and is a different fact from "never + # shared". Without the owner the column is unmanageable: `may_administer` fails closed + # on an ownerless record, so nobody could ever share it. + shares.set_grants("field", shares.field_oid(SHARE_TOPIC, key), [], + owner=session.uname, st=session.runtime) + except Exception: # noqa: BLE001 + # The mark is already written, so a failed claim fails CLOSED: governed, nobody + # granted, admin-and-creator only. Recoverable. The other order is not. + pass + + # ⛔ AND THE RECORDS THAT LOST THEIR NUMBER ARE CLEARED. A re-solve over a SMALLER cohort + # would otherwise leave the previous run's ranks sitting on the records that dropped out — + # plausible integers, from a route nobody is driving. `""` is the house spelling of an empty + # overlay cell (`rows_from_pool` defaults an absent one to exactly that), so this needs no new + # vocabulary and no tombstone nobody else reads. + stale = {} + for raw_pid, cells in (shared_cells(pool, st=session.runtime) or {}).items(): + if not isinstance(cells, dict) or cells.get(key) in (None, ""): + continue + try: + gone = int(raw_pid) + except (TypeError, ValueError): + continue + if gone not in ranks: + stale[gone] = {key: ""} + written = shared_overlay.put_rows( + _shared_key(), {**{p: {key: v} for p, v in ranks.items()}, **stale}, + st=session.runtime) + + out = {"ok": True, "field": key, "label": defn["label"], "type": "int", + "stops": len(ranks), "cleared": len(stale), + "rows_written": len(written), "inputsHash": defn["route"]["inputsHash"], + "solvedAt": stamp} + return out diff --git a/api/routes_geo.py b/api/routes_geo.py index 25575695d4a054d4e567113792da75a94b75a56a..c0b7d2c67d135350f881a36d466f9b767ad34824 100644 --- a/api/routes_geo.py +++ b/api/routes_geo.py @@ -1,369 +1,369 @@ -"""routes_geo.py — the MAP PROVIDER SEAM (wave 37, R3/R14, tickets T35/T36/T37). - -⭐ WHY THIS FILE EXISTS AT ALL, and why the map does not simply call a vendor. - -R14 (owner, 2026-08-19): no Google Maps API key this wave. The map work is not dropped, it is -retargeted to providers that need no key: OpenStreetMap tiles, OSRM routing, Nominatim geocoding. -The deferral only holds if swapping a paid vendor back in is a CONFIG change rather than a rewrite, -so every provider is a (base URL + attribution) pair read from the environment and served from ONE -door. The renderer receives a URL template and a credit line; it knows nothing about who is behind -them. That is the seam. - - tiles AIOS_MAP_TILE_URL default https://tile.openstreetmap.org/{z}/{x}/{y}.png - geocode AIOS_GEOCODE_URL default https://nominatim.openstreetmap.org/search - routing AIOS_ROUTE_URL default https://router.project-osrm.org/route/v1/driving/ - -⛔ R3'S ON-DEMAND RULE, AND WHY IT IS STRUCTURAL HERE RATHER THAN A COMMENT. -R3 forbids geocoding a whole table automatically. Under R14 the reason moves from money to manners -and gets sharper: Nominatim's usage policy allows roughly one request per second and explicitly -forbids bulk harvesting, so a loop over 3,636 customers does not produce a bill, it produces a -BLOCKED tenant. Three mechanisms enforce it, none of them advisory: - - 1. `/geo/geocode` takes an explicit LIST OF ADDRESSES from the caller. It cannot read a table, so - there is no code path from "a database exists" to "its rows were geocoded". Somebody had to - choose the records. - 2. `MAX_BATCH` refuses an oversized list with a named error rather than truncating it. The client - drives the loop and can therefore SHOW the wait, which is the half of the ticket a comment - cannot satisfy. - 3. `_throttle()` blocks in-process until the minimum interval has elapsed, so even a caller that - ignores everything above cannot exceed the published rate. - -⭐ AND A CACHE, because the policy asks for one: an address geocoded once is answered from memory -for the rest of the container's life. It is the cheapest way to be a good citizen and it makes a -re-run of the same records free rather than merely legal. - -⚠ NOTHING HERE PROXIES TILES. Tiles are fetched by the BROWSER, straight from the provider, which -is what every OSM client does and what the tile policy expects. Proxying them through this app -would put a free tier in the path of every pan and would breach the same policy it looks like it is -respecting. This door serves the tile URL, never the tile. -""" -import os -import re -import threading -import time -from collections import OrderedDict - -import requests -from fastapi import APIRouter, Body, Depends - -from deps import Session, err, require_session - -router = APIRouter(prefix="/api/v1") - -# --------------------------------------------------------------------- the seam -# -# Every value below is an environment override with a keyless default. Pointing the map at a paid -# vendor is three variables on the Space and no code change, which is what makes R14 a deferral. - -_TILE_URL_DEFAULT = "https://tile.openstreetmap.org/{z}/{x}/{y}.png" -_TILE_CREDIT_DEFAULT = "© OpenStreetMap contributors" -_TILE_CREDIT_HREF_DEFAULT = "https://www.openstreetmap.org/copyright" -_GEOCODE_URL_DEFAULT = "https://nominatim.openstreetmap.org/search" -_ROUTE_URL_DEFAULT = "https://router.project-osrm.org/route/v1/driving/" - -#: Both policies require a real, identifying User-Agent. A generic one is how a shared service -#: decides an application is a scraper, so this carries the product name and a contact URL. -_UA_DEFAULT = "AIOS-Loopable/1.0 (+https://runloopable.com)" - -#: Nominatim publishes one request per second. 1100 ms leaves room for clock jitter rather than -#: sitting exactly on the published edge. -_MIN_INTERVAL_MS_DEFAULT = 1100 - -#: The most addresses one call may carry. Small on purpose: the client loops and shows progress, -#: and no single request can sit on the connection for a minute waiting out the throttle. -MAX_BATCH = 25 - -#: Tile zoom the provider actually serves. OSM stops at 19. -_TILE_MAX_Z_DEFAULT = 19 - -_CACHE_MAX = 4096 - - -def _env(name, default): - return (os.environ.get(name) or "").strip() or default - - -def _env_int(name, default): - try: - n = int((os.environ.get(name) or "").strip()) - return n if n > 0 else default - except (TypeError, ValueError): - return default - - -def provider_config(): - """The whole seam as one dict. `GET /geo/providers` serves it and the gate reads it.""" - return { - "tiles": { - "url": _env("AIOS_MAP_TILE_URL", _TILE_URL_DEFAULT), - "attribution": _env("AIOS_MAP_TILE_ATTRIBUTION", _TILE_CREDIT_DEFAULT), - "attributionUrl": _env("AIOS_MAP_TILE_ATTRIBUTION_URL", _TILE_CREDIT_HREF_DEFAULT), - "maxZoom": _env_int("AIOS_MAP_TILE_MAX_Z", _TILE_MAX_Z_DEFAULT), - }, - "geocode": { - "available": True, - "maxBatch": MAX_BATCH, - "minIntervalMs": _env_int("AIOS_GEOCODE_MIN_INTERVAL_MS", _MIN_INTERVAL_MS_DEFAULT), - # ⭐⭐ THE NUMBER THE SURFACE MUST QUOTE, AND IT IS NOT THE RATE LIMIT. - # T37 asks for the estimated wait to be shown BEFORE a run. The obvious source is - # `minIntervalMs`, and it is wrong: 1.1 s is a floor on POLITENESS, not a prediction of - # LATENCY. One real lookup was MEASURED at 3.28 s end to end on 2026-08-19. Quoting the - # limit would promise a minute and take three, and a progress estimate that runs out - # before the work does reads as a hang rather than as a wait. - "secondsPerAddress": float(_env("AIOS_GEOCODE_SECONDS_EACH", "3.3")), - "attribution": _env("AIOS_GEOCODE_ATTRIBUTION", _TILE_CREDIT_DEFAULT), - }, - "route": { - "available": True, - "attribution": _env("AIOS_ROUTE_ATTRIBUTION", "Routing by OSRM"), - }, - } - - -# ------------------------------------------------------------------ the throttle -# -# ⚠ ONE lock and ONE timestamp for the whole process, shared by geocoding and routing, because the -# rate limit belongs to the SERVICE and not to the endpoint. Two independent throttles would each -# stay honest and together double the published rate. - -_gate = threading.Lock() -_last_call = [0.0] - - -def _throttle(min_interval_ms): - """Block until the minimum interval since the last outbound call has elapsed. - - Returns the seconds actually waited, so a caller can report the wait rather than hide it.""" - wait = 0.0 - with _gate: - gap = min_interval_ms / 1000.0 - now = time.monotonic() - due = _last_call[0] + gap - if now < due: - wait = due - now - time.sleep(wait) - _last_call[0] = time.monotonic() - return wait - - -# --------------------------------------------------------------------- the cache - -_cache = OrderedDict() -_cache_lock = threading.Lock() - - -def _cache_key(q, country): - return (re.sub(r"\s+", " ", str(q or "")).strip().lower(), str(country or "").strip().lower()) - - -def _cache_get(key): - with _cache_lock: - if key not in _cache: - return None - _cache.move_to_end(key) - return _cache[key] - - -def _cache_put(key, value): - with _cache_lock: - _cache[key] = value - _cache.move_to_end(key) - while len(_cache) > _CACHE_MAX: - _cache.popitem(last=False) - - -def geocode_one(address, country=None, timeout=12): - """One address to {lat, lon, label} or None. - - ⛔ THE ONLY PLACE AN OUTBOUND GEOCODE HAPPENS. `ai_enrich.py`'s geocode field calls this rather - than reaching for `requests` itself, so the throttle and the cache cannot be walked around by a - second caller. A cache hit costs no request and no wait.""" - q = re.sub(r"\s+", " ", str(address or "")).strip() - if not q: - return None - key = _cache_key(q, country) - hit = _cache_get(key) - if hit is not None: - return dict(hit) if hit else None - - cfg = provider_config()["geocode"] - _throttle(cfg["minIntervalMs"]) - params = {"q": q, "format": "jsonv2", "limit": 1} - if country: - params["countrycodes"] = str(country).strip().lower() - try: - r = requests.get( - _env("AIOS_GEOCODE_URL", _GEOCODE_URL_DEFAULT), - params=params, - headers={"User-Agent": _env("AIOS_GEO_USER_AGENT", _UA_DEFAULT), - "Accept": "application/json"}, - timeout=timeout, - ) - r.raise_for_status() - rows = r.json() - except Exception: - # ⚠ A transport failure is NOT cached. Caching it would turn one bad minute into a - # permanently empty column that no re-run could ever repair. - return None - if not isinstance(rows, list) or not rows: - # A genuine "no such place" IS cached, as a negative: asking again gets the same answer and - # spends another second of a shared service's budget to hear it. - _cache_put(key, {}) - return None - top = rows[0] or {} - try: - lat = float(top.get("lat")) - lon = float(top.get("lon")) - except (TypeError, ValueError): - _cache_put(key, {}) - return None - if not (-90 <= lat <= 90) or not (-180 <= lon <= 180): - _cache_put(key, {}) - return None - out = {"lat": lat, "lon": lon, "label": str(top.get("display_name") or q)} - _cache_put(key, out) - return dict(out) - - -# ---------------------------------------------------------------------- the doors - - -@router.get("/geo/providers") -def geo_providers(session: Session = Depends(require_session)): - """What the map should draw with, and who to credit for it. - - Session gated like every other door here. The values are not secret, but an unauthenticated - endpoint on this app is a door somebody eventually hangs something else on.""" - return provider_config() - - -@router.post("/geo/geocode") -def geo_geocode(body: dict = Body(...), session: Session = Depends(require_session)): - """Turn a bounded list of addresses into coordinates. - - ⛔ Takes ADDRESSES, never a table key, a view id or a filter. There is deliberately no shape of - request that means "geocode everything", which is R3 expressed as an API rather than as a - warning.""" - items = body.get("addresses") - # ⭐ THE SECOND SHAPE, AND IT EXISTS TO KEEP ONE COMPOSER. A caller may send whole RECORDS plus - # the geocode field's config instead of finished address strings, and the server builds the - # query with `ai_enrich.geocode_query`. The alternative was a TypeScript twin of that function - # on the client, and an address composer that exists twice will disagree the first time either - # copy learns about a new column ([[one-question-two-normalizers]]) -- which is exactly the trap - # B's measured `New York (US)` suffix would spring, silently, in whichever copy forgot. - if items is None: - records = body.get("records") - cfg = body.get("config") or {} - if isinstance(records, list) and records: - import ai_enrich - items = [] - for rec in records: - if not isinstance(rec, dict): - continue - q = ai_enrich.geocode_query(rec.get("row") or {}, cfg) - # ⛔ A record with no usable address is REPORTED, not dropped: it comes back - # `found: false` with an empty address, so the caller can name it on screen instead - # of quietly returning fewer answers than it asked questions. - items.append({"key": rec.get("key"), "address": q}) - if not body.get("country") and cfg.get("country"): - body = {**body, "country": cfg.get("country")} - if not isinstance(items, list) or not items: - raise err(400, "no_addresses", "Send at least one address to look up.") - if len(items) > MAX_BATCH: - raise err( - 400, - "batch_too_large", - f"Look up at most {MAX_BATCH} addresses per request. " - f"The map sends them in batches of {MAX_BATCH} so the wait stays visible.", - ) - country = body.get("country") - out = [] - started = time.monotonic() - for raw in items: - if isinstance(raw, dict): - key, addr = raw.get("key"), raw.get("address") - else: - key, addr = None, raw - # ⛔ AN EMPTY QUERY COSTS NO REQUEST AND CARRIES ITS OWN REASON. Nominatim's policy is a - # budget shared with everyone else using it, and asking it to place "" would spend a second - # of that budget to be told nothing. `reason` is what lets the surface say "no address on - # file" rather than "not found", which are different facts and want different actions. - if not str(addr or "").strip(): - out.append({"key": key, "address": "", "found": False, "lat": None, "lon": None, - "label": None, "reason": "no_address"}) - continue - hit = geocode_one(addr, country=country) - out.append({"key": key, "address": str(addr or ""), "found": bool(hit), - "reason": None if hit else "not_found", - "lat": hit["lat"] if hit else None, - "lon": hit["lon"] if hit else None, - "label": hit["label"] if hit else None}) - elapsed = time.monotonic() - started - return {"results": out, - "found": sum(1 for r in out if r["found"]), - "asked": len(out), - "secondsElapsed": round(elapsed, 2), - "attribution": provider_config()["geocode"]["attribution"]} - - -@router.post("/geo/route") -def geo_route(body: dict = Body(...), session: Session = Depends(require_session)): - """Road distance, duration and the drawn line, for stops ALREADY put in order. - - ⭐ THE ORDERING IS NOT DONE HERE. `mapProjection.planRoute` sequences the stops on the client, - for free, and keeps working when this service does not answer. This door adds the half that - arithmetic cannot produce: what the ROADS actually cost. Splitting it that way is why the Start - picker and the round-trip toggle keep working with no network at all.""" - stops = body.get("stops") - if not isinstance(stops, list) or len(stops) < 2: - raise err(400, "too_few_stops", "A route needs at least two stops with a location.") - if len(stops) > 25: - raise err(400, "too_many_stops", - "Route at most 25 stops at once. Narrow the selection and try again.") - pairs = [] - for s in stops: - try: - lat, lon = float(s["lat"]), float(s["lon"]) - except (TypeError, ValueError, KeyError, IndexError): - raise err(400, "bad_stop", "Every stop needs a numeric latitude and longitude.") - if not (-90 <= lat <= 90) or not (-180 <= lon <= 180): - raise err(400, "bad_stop", "Every stop needs a latitude and longitude on the globe.") - pairs.append(f"{lon:.6f},{lat:.6f}") - - base = _env("AIOS_ROUTE_URL", _ROUTE_URL_DEFAULT) - _throttle(_env_int("AIOS_GEOCODE_MIN_INTERVAL_MS", _MIN_INTERVAL_MS_DEFAULT)) - try: - r = requests.get( - base.rstrip("/") + "/" + ";".join(pairs), - params={"overview": "simplified", "geometries": "geojson", "steps": "false"}, - headers={"User-Agent": _env("AIOS_GEO_USER_AGENT", _UA_DEFAULT)}, - timeout=20, - ) - r.raise_for_status() - data = r.json() - except Exception: - raise err(502, "route_service_unavailable", - "The routing service did not answer. The stop order and the straight line " - "distance are still on the map.") - routes = (data or {}).get("routes") or [] - if not routes: - raise err( - 502, - "no_route", - "No road route connects these stops. They may be on different land masses, or one of " - "them may be far from any road.", - ) - top = routes[0] - geom = ((top.get("geometry") or {}).get("coordinates")) or [] - line = [] - for c in geom: - try: - line.append([float(c[0]), float(c[1])]) - except (TypeError, ValueError, IndexError): - continue - return { - "km": round(float(top.get("distance") or 0.0) / 1000.0, 1), - "minutes": int(round(float(top.get("duration") or 0.0) / 60.0)), - "line": line, - "attribution": provider_config()["route"]["attribution"], - } +"""routes_geo.py — the MAP PROVIDER SEAM (wave 37, R3/R14, tickets T35/T36/T37). + +⭐ WHY THIS FILE EXISTS AT ALL, and why the map does not simply call a vendor. + +R14 (owner, 2026-08-19): no Google Maps API key this wave. The map work is not dropped, it is +retargeted to providers that need no key: OpenStreetMap tiles, OSRM routing, Nominatim geocoding. +The deferral only holds if swapping a paid vendor back in is a CONFIG change rather than a rewrite, +so every provider is a (base URL + attribution) pair read from the environment and served from ONE +door. The renderer receives a URL template and a credit line; it knows nothing about who is behind +them. That is the seam. + + tiles AIOS_MAP_TILE_URL default https://tile.openstreetmap.org/{z}/{x}/{y}.png + geocode AIOS_GEOCODE_URL default https://nominatim.openstreetmap.org/search + routing AIOS_ROUTE_URL default https://router.project-osrm.org/route/v1/driving/ + +⛔ R3'S ON-DEMAND RULE, AND WHY IT IS STRUCTURAL HERE RATHER THAN A COMMENT. +R3 forbids geocoding a whole table automatically. Under R14 the reason moves from money to manners +and gets sharper: Nominatim's usage policy allows roughly one request per second and explicitly +forbids bulk harvesting, so a loop over 3,636 customers does not produce a bill, it produces a +BLOCKED tenant. Three mechanisms enforce it, none of them advisory: + + 1. `/geo/geocode` takes an explicit LIST OF ADDRESSES from the caller. It cannot read a table, so + there is no code path from "a database exists" to "its rows were geocoded". Somebody had to + choose the records. + 2. `MAX_BATCH` refuses an oversized list with a named error rather than truncating it. The client + drives the loop and can therefore SHOW the wait, which is the half of the ticket a comment + cannot satisfy. + 3. `_throttle()` blocks in-process until the minimum interval has elapsed, so even a caller that + ignores everything above cannot exceed the published rate. + +⭐ AND A CACHE, because the policy asks for one: an address geocoded once is answered from memory +for the rest of the container's life. It is the cheapest way to be a good citizen and it makes a +re-run of the same records free rather than merely legal. + +⚠ NOTHING HERE PROXIES TILES. Tiles are fetched by the BROWSER, straight from the provider, which +is what every OSM client does and what the tile policy expects. Proxying them through this app +would put a free tier in the path of every pan and would breach the same policy it looks like it is +respecting. This door serves the tile URL, never the tile. +""" +import os +import re +import threading +import time +from collections import OrderedDict + +import requests +from fastapi import APIRouter, Body, Depends + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + +# --------------------------------------------------------------------- the seam +# +# Every value below is an environment override with a keyless default. Pointing the map at a paid +# vendor is three variables on the Space and no code change, which is what makes R14 a deferral. + +_TILE_URL_DEFAULT = "https://tile.openstreetmap.org/{z}/{x}/{y}.png" +_TILE_CREDIT_DEFAULT = "© OpenStreetMap contributors" +_TILE_CREDIT_HREF_DEFAULT = "https://www.openstreetmap.org/copyright" +_GEOCODE_URL_DEFAULT = "https://nominatim.openstreetmap.org/search" +_ROUTE_URL_DEFAULT = "https://router.project-osrm.org/route/v1/driving/" + +#: Both policies require a real, identifying User-Agent. A generic one is how a shared service +#: decides an application is a scraper, so this carries the product name and a contact URL. +_UA_DEFAULT = "AIOS-Loopable/1.0 (+https://runloopable.com)" + +#: Nominatim publishes one request per second. 1100 ms leaves room for clock jitter rather than +#: sitting exactly on the published edge. +_MIN_INTERVAL_MS_DEFAULT = 1100 + +#: The most addresses one call may carry. Small on purpose: the client loops and shows progress, +#: and no single request can sit on the connection for a minute waiting out the throttle. +MAX_BATCH = 25 + +#: Tile zoom the provider actually serves. OSM stops at 19. +_TILE_MAX_Z_DEFAULT = 19 + +_CACHE_MAX = 4096 + + +def _env(name, default): + return (os.environ.get(name) or "").strip() or default + + +def _env_int(name, default): + try: + n = int((os.environ.get(name) or "").strip()) + return n if n > 0 else default + except (TypeError, ValueError): + return default + + +def provider_config(): + """The whole seam as one dict. `GET /geo/providers` serves it and the gate reads it.""" + return { + "tiles": { + "url": _env("AIOS_MAP_TILE_URL", _TILE_URL_DEFAULT), + "attribution": _env("AIOS_MAP_TILE_ATTRIBUTION", _TILE_CREDIT_DEFAULT), + "attributionUrl": _env("AIOS_MAP_TILE_ATTRIBUTION_URL", _TILE_CREDIT_HREF_DEFAULT), + "maxZoom": _env_int("AIOS_MAP_TILE_MAX_Z", _TILE_MAX_Z_DEFAULT), + }, + "geocode": { + "available": True, + "maxBatch": MAX_BATCH, + "minIntervalMs": _env_int("AIOS_GEOCODE_MIN_INTERVAL_MS", _MIN_INTERVAL_MS_DEFAULT), + # ⭐⭐ THE NUMBER THE SURFACE MUST QUOTE, AND IT IS NOT THE RATE LIMIT. + # T37 asks for the estimated wait to be shown BEFORE a run. The obvious source is + # `minIntervalMs`, and it is wrong: 1.1 s is a floor on POLITENESS, not a prediction of + # LATENCY. One real lookup was MEASURED at 3.28 s end to end on 2026-08-19. Quoting the + # limit would promise a minute and take three, and a progress estimate that runs out + # before the work does reads as a hang rather than as a wait. + "secondsPerAddress": float(_env("AIOS_GEOCODE_SECONDS_EACH", "3.3")), + "attribution": _env("AIOS_GEOCODE_ATTRIBUTION", _TILE_CREDIT_DEFAULT), + }, + "route": { + "available": True, + "attribution": _env("AIOS_ROUTE_ATTRIBUTION", "Routing by OSRM"), + }, + } + + +# ------------------------------------------------------------------ the throttle +# +# ⚠ ONE lock and ONE timestamp for the whole process, shared by geocoding and routing, because the +# rate limit belongs to the SERVICE and not to the endpoint. Two independent throttles would each +# stay honest and together double the published rate. + +_gate = threading.Lock() +_last_call = [0.0] + + +def _throttle(min_interval_ms): + """Block until the minimum interval since the last outbound call has elapsed. + + Returns the seconds actually waited, so a caller can report the wait rather than hide it.""" + wait = 0.0 + with _gate: + gap = min_interval_ms / 1000.0 + now = time.monotonic() + due = _last_call[0] + gap + if now < due: + wait = due - now + time.sleep(wait) + _last_call[0] = time.monotonic() + return wait + + +# --------------------------------------------------------------------- the cache + +_cache = OrderedDict() +_cache_lock = threading.Lock() + + +def _cache_key(q, country): + return (re.sub(r"\s+", " ", str(q or "")).strip().lower(), str(country or "").strip().lower()) + + +def _cache_get(key): + with _cache_lock: + if key not in _cache: + return None + _cache.move_to_end(key) + return _cache[key] + + +def _cache_put(key, value): + with _cache_lock: + _cache[key] = value + _cache.move_to_end(key) + while len(_cache) > _CACHE_MAX: + _cache.popitem(last=False) + + +def geocode_one(address, country=None, timeout=12): + """One address to {lat, lon, label} or None. + + ⛔ THE ONLY PLACE AN OUTBOUND GEOCODE HAPPENS. `ai_enrich.py`'s geocode field calls this rather + than reaching for `requests` itself, so the throttle and the cache cannot be walked around by a + second caller. A cache hit costs no request and no wait.""" + q = re.sub(r"\s+", " ", str(address or "")).strip() + if not q: + return None + key = _cache_key(q, country) + hit = _cache_get(key) + if hit is not None: + return dict(hit) if hit else None + + cfg = provider_config()["geocode"] + _throttle(cfg["minIntervalMs"]) + params = {"q": q, "format": "jsonv2", "limit": 1} + if country: + params["countrycodes"] = str(country).strip().lower() + try: + r = requests.get( + _env("AIOS_GEOCODE_URL", _GEOCODE_URL_DEFAULT), + params=params, + headers={"User-Agent": _env("AIOS_GEO_USER_AGENT", _UA_DEFAULT), + "Accept": "application/json"}, + timeout=timeout, + ) + r.raise_for_status() + rows = r.json() + except Exception: + # ⚠ A transport failure is NOT cached. Caching it would turn one bad minute into a + # permanently empty column that no re-run could ever repair. + return None + if not isinstance(rows, list) or not rows: + # A genuine "no such place" IS cached, as a negative: asking again gets the same answer and + # spends another second of a shared service's budget to hear it. + _cache_put(key, {}) + return None + top = rows[0] or {} + try: + lat = float(top.get("lat")) + lon = float(top.get("lon")) + except (TypeError, ValueError): + _cache_put(key, {}) + return None + if not (-90 <= lat <= 90) or not (-180 <= lon <= 180): + _cache_put(key, {}) + return None + out = {"lat": lat, "lon": lon, "label": str(top.get("display_name") or q)} + _cache_put(key, out) + return dict(out) + + +# ---------------------------------------------------------------------- the doors + + +@router.get("/geo/providers") +def geo_providers(session: Session = Depends(require_session)): + """What the map should draw with, and who to credit for it. + + Session gated like every other door here. The values are not secret, but an unauthenticated + endpoint on this app is a door somebody eventually hangs something else on.""" + return provider_config() + + +@router.post("/geo/geocode") +def geo_geocode(body: dict = Body(...), session: Session = Depends(require_session)): + """Turn a bounded list of addresses into coordinates. + + ⛔ Takes ADDRESSES, never a table key, a view id or a filter. There is deliberately no shape of + request that means "geocode everything", which is R3 expressed as an API rather than as a + warning.""" + items = body.get("addresses") + # ⭐ THE SECOND SHAPE, AND IT EXISTS TO KEEP ONE COMPOSER. A caller may send whole RECORDS plus + # the geocode field's config instead of finished address strings, and the server builds the + # query with `ai_enrich.geocode_query`. The alternative was a TypeScript twin of that function + # on the client, and an address composer that exists twice will disagree the first time either + # copy learns about a new column ([[one-question-two-normalizers]]) -- which is exactly the trap + # B's measured `New York (US)` suffix would spring, silently, in whichever copy forgot. + if items is None: + records = body.get("records") + cfg = body.get("config") or {} + if isinstance(records, list) and records: + import ai_enrich + items = [] + for rec in records: + if not isinstance(rec, dict): + continue + q = ai_enrich.geocode_query(rec.get("row") or {}, cfg) + # ⛔ A record with no usable address is REPORTED, not dropped: it comes back + # `found: false` with an empty address, so the caller can name it on screen instead + # of quietly returning fewer answers than it asked questions. + items.append({"key": rec.get("key"), "address": q}) + if not body.get("country") and cfg.get("country"): + body = {**body, "country": cfg.get("country")} + if not isinstance(items, list) or not items: + raise err(400, "no_addresses", "Send at least one address to look up.") + if len(items) > MAX_BATCH: + raise err( + 400, + "batch_too_large", + f"Look up at most {MAX_BATCH} addresses per request. " + f"The map sends them in batches of {MAX_BATCH} so the wait stays visible.", + ) + country = body.get("country") + out = [] + started = time.monotonic() + for raw in items: + if isinstance(raw, dict): + key, addr = raw.get("key"), raw.get("address") + else: + key, addr = None, raw + # ⛔ AN EMPTY QUERY COSTS NO REQUEST AND CARRIES ITS OWN REASON. Nominatim's policy is a + # budget shared with everyone else using it, and asking it to place "" would spend a second + # of that budget to be told nothing. `reason` is what lets the surface say "no address on + # file" rather than "not found", which are different facts and want different actions. + if not str(addr or "").strip(): + out.append({"key": key, "address": "", "found": False, "lat": None, "lon": None, + "label": None, "reason": "no_address"}) + continue + hit = geocode_one(addr, country=country) + out.append({"key": key, "address": str(addr or ""), "found": bool(hit), + "reason": None if hit else "not_found", + "lat": hit["lat"] if hit else None, + "lon": hit["lon"] if hit else None, + "label": hit["label"] if hit else None}) + elapsed = time.monotonic() - started + return {"results": out, + "found": sum(1 for r in out if r["found"]), + "asked": len(out), + "secondsElapsed": round(elapsed, 2), + "attribution": provider_config()["geocode"]["attribution"]} + + +@router.post("/geo/route") +def geo_route(body: dict = Body(...), session: Session = Depends(require_session)): + """Road distance, duration and the drawn line, for stops ALREADY put in order. + + ⭐ THE ORDERING IS NOT DONE HERE. `mapProjection.planRoute` sequences the stops on the client, + for free, and keeps working when this service does not answer. This door adds the half that + arithmetic cannot produce: what the ROADS actually cost. Splitting it that way is why the Start + picker and the round-trip toggle keep working with no network at all.""" + stops = body.get("stops") + if not isinstance(stops, list) or len(stops) < 2: + raise err(400, "too_few_stops", "A route needs at least two stops with a location.") + if len(stops) > 25: + raise err(400, "too_many_stops", + "Route at most 25 stops at once. Narrow the selection and try again.") + pairs = [] + for s in stops: + try: + lat, lon = float(s["lat"]), float(s["lon"]) + except (TypeError, ValueError, KeyError, IndexError): + raise err(400, "bad_stop", "Every stop needs a numeric latitude and longitude.") + if not (-90 <= lat <= 90) or not (-180 <= lon <= 180): + raise err(400, "bad_stop", "Every stop needs a latitude and longitude on the globe.") + pairs.append(f"{lon:.6f},{lat:.6f}") + + base = _env("AIOS_ROUTE_URL", _ROUTE_URL_DEFAULT) + _throttle(_env_int("AIOS_GEOCODE_MIN_INTERVAL_MS", _MIN_INTERVAL_MS_DEFAULT)) + try: + r = requests.get( + base.rstrip("/") + "/" + ";".join(pairs), + params={"overview": "simplified", "geometries": "geojson", "steps": "false"}, + headers={"User-Agent": _env("AIOS_GEO_USER_AGENT", _UA_DEFAULT)}, + timeout=20, + ) + r.raise_for_status() + data = r.json() + except Exception: + raise err(502, "route_service_unavailable", + "The routing service did not answer. The stop order and the straight line " + "distance are still on the map.") + routes = (data or {}).get("routes") or [] + if not routes: + raise err( + 502, + "no_route", + "No road route connects these stops. They may be on different land masses, or one of " + "them may be far from any road.", + ) + top = routes[0] + geom = ((top.get("geometry") or {}).get("coordinates")) or [] + line = [] + for c in geom: + try: + line.append([float(c[0]), float(c[1])]) + except (TypeError, ValueError, IndexError): + continue + return { + "km": round(float(top.get("distance") or 0.0) / 1000.0, 1), + "minutes": int(round(float(top.get("duration") or 0.0) / 60.0)), + "line": line, + "attribution": provider_config()["route"]["attribution"], + } diff --git a/api/routes_grid.py b/api/routes_grid.py index 2a20938bf44785ba2b56b9206f4299803ea6e736..27cec4519f43ae113fb2a06b685bff9fbeaceb7c 100644 --- a/api/routes_grid.py +++ b/api/routes_grid.py @@ -1,1083 +1,1083 @@ -"""routes_grid.py — X2's write seam over HTTP: the SECOND adapter on `core.grid_events` (EXIT-1c). - -`POST /api/v1/grid/events` takes the component's event objects VERBATIM — the same objects the -Streamlit host receives through the component value slot, unchanged — and runs them through the -same `core.grid_events.handle_events` the Streamlit adapter runs. That is the whole point of -EXIT-1a: one implementation of every permission wall, two transports. A route that re-validated -anything here would be a second wall to keep in step, and the two would drift on the first change. - -DEDUP IS PER REQUEST, and that is a deliberate limit, not an oversight. The Streamlit adapter's -`seen_ids` lives in `st.session_state` — genuinely per user session — because the client resends -its recent 24-event window on every emit inside one page session. A stateless API has no such -dict, and inventing a per-user server-side one would be exactly the resident per-tenant state -EXIT-4a exists to remove. So: ids are deduped WITHIN a request body (the resend window's whole -purpose — a batch that repeats an id processes it once), and a genuinely replayed request is -handled by the operations being idempotent. The one event where a replay is observable is -`add_to_list` (it would add the same pids to the same cohort twice — a set union, so the -membership is unchanged, but the toast count repeats). Noted rather than papered over; a -server-side idempotency key belongs with D2's session mirror (C-2). - -STORE DOWN = 503. `fallback_ws` is None on this adapter, so `core.grid_events` raises -`StoreUnavailable` rather than writing to an in-memory workspace no API request could ever read -back. A 200 over a write that evaporated is the failure this rule exists to prevent. -""" -import datetime as dt -import time - -from fastapi import APIRouter, Body, Depends - -from deps import Session, err, module_gate, perms, require_session - -router = APIRouter(prefix="/api/v1") - -MODULE = "customer_data" - -#: The client's own resend window (`app.py`'s `[-24:]`). A body larger than this is not a -#: legitimate client — refuse it rather than doing 500 store writes on one request. -_MAX_EVENTS = 24 - - -def _ctx(session: Session, fields, pids, **kw): - from core import grid_events - import aios_grid - import core.perm_scope as perm_scope - - # Wave 16 C-TOPIC: the ctx is TOPIC-SHAPED. The product scope swaps all three of the - # things a write is validated against — the module the permission wall reads, the canonical - # contract the hidden-field closure runs over, and the TABLE OPS the write lands in. - # Getting any one of them from the other topic is the "validated against the wrong field - # contract" near-miss the wave-15 routes_products header warned about. - scope_key = str(kw.get("scope_key") or "") - if scope_key == "product": - import modules.product_data as pd - from routes_products import MODULE as _PMOD, pd_fields - - module, canonical = _PMOD, pd_fields(consolidated=True) - kw.setdefault("table", pd.TABLE_OPS) - hidden = perm_scope.hidden_keys(session.user, module, canonical) - elif scope_key.startswith("ut_"): - # Wave 18 C3-UT: a user table has no module in the permission wall (its wall is - # `user_tables.may_open`, already applied by the assembly this route ran first), so - # the hidden-field closure is EMPTY rather than borrowed from another topic's contract. - import core.table_store as table_store - - kw.setdefault("table", - table_store.make(f"{scope_key}_table_workspace", st=session.runtime)) - hidden = frozenset() - else: - module, canonical = MODULE, aios_grid.FIELDS - hidden = perm_scope.hidden_keys(session.user, module, canonical) - return grid_events.EventCtx( - uname=session.uname, allowed_pids=pids, fields=fields, admin=session.admin, - # C-PERM: the write wall's field half, on the EVENTS transport too. This route takes the - # component's event objects verbatim, so a hidden key would otherwise arrive here with - # nothing between it and the store. - hidden_keys=hidden, - # ⭐ WAVE 25 (R6b, closes D-16) — THE TENANT HANDLE, on EVERY scope. This is the line - # D-16's exit condition names, and without it the rest of the fix is inert: the handler - # falls back to the module-global `core.store`, which is tenant #0's repo, so a Nurilab - # user's documents, cohorts and — through `_tops` — their whole customer/product table - # workspace were written into Royal Imports' dataset. Note it is set for the CUSTOMER - # and PRODUCT branches too, not only `ut_`: those two never passed a scoped `table`, so - # they were the ones actually resolving to tenant #0 on every request. - st=session.runtime, - fallback_ws=None, seen_ids={}, **kw) - - -#: The surfaces the ONE grid serves. `cohort` is the same table over hand-curated SETS rather -#: than over the whole scoped pool — `app.py:page_cohort` calls the same `_table_grid` with -#: `cohort_mode=True, scope_key='cohort'`. `product` (wave 16 C-TOPIC) is the SKU table: -#: same engine, its own pool, field contract and workspace BUCKET (routes_products). -_SCOPES = ("customer", "cohort", "product") - - -def _scope_or_400(raw): - """⛔ REFUSE AN UNKNOWN SCOPE, never default it. A typo silently served as `customer` would - hand a user the whole book on a page they opened to see one cohort — the widening direction, - which is the one that must fail closed. - - Wave 18 (C3-UT): a `ut_`-prefixed scope names a USER TABLE and passes through here — - existence and the per-table wall are enforced by `routes_tables.ut_assembly` (404/403), - which every consumer of such a scope goes through. Passing an unknown ut key therefore - still fails closed, just one layer down where the store can actually be consulted.""" - scope = (raw or "customer").strip().lower() - if scope.startswith("ut_"): - return scope - if scope not in _SCOPES: - raise err(400, "bad_scope", - f"scope must be one of {', '.join(_SCOPES)} — refusing to guess") - return scope - - -@router.get("/workspace") -def workspace(scope: str = "customer", - session: Session = Depends(require_session)): - """The durable table workspace for this session: views, fields, overlays, folders. - - ⛔ WAVE 21 (C4): the wall is TOPIC-SHAPED, so the DEPENDENCY is session-only and each scope - asserts its own gate below. The old `module_gate("customer_data")` dependency 403'd a - `ut_*` workspace for any tenant whose catalogue omits the customer module (loopable/ - nurilab/gtmlab ship modules w/o it) — and the client then fell back to the shared - localStorage bucket + demo data, which is exactly the "new database shows RI's customer - fields" defect. A user table's real wall is `user_tables.may_open`, enforced inside - `ut_assembly` (404/403, one layer down where the store can be consulted). - - ⚠ WHY THIS EXISTS (S1↔S2, 2026-07-30 — an X2 AMENDMENT, see the split doc). X2 fixes - `/customers` at the exact shape `verify_fields_contract.py` referees, so the workspace cannot - ride along in it without forking the thing that gate keeps single-sourced. Without a - workspace route the standalone shell's write path would be WRITE-ONLY: events persist - server-side and nothing reads them back on reload, so a saved view looks lost to the user - even though the store has it. This route closes that read-back gap at its own URL. - - `allowed_pids` is passed so a SHARED view's `memberPids` are re-scoped to THIS reader — the - wave-9 leak rule. Omitting it would hand a Fisch-scoped user a member list built by a - full-access user. - """ - from core import grid_events - from routes_customers import grid_assembly - - scope = _scope_or_400(scope) - # Wave 21 C4 — the per-scope gate the dependency no longer asserts: customer/cohort need - # the customer module; product asserts PRODUCT_MODULE in its branch; ut_* needs only a - # session (its wall is the table's own, inside the assembly). - if not (scope == "product" or scope.startswith("ut_")): - session.require(MODULE) - # ── Wave 18 C3-UT: a USER TABLE'S workspace — the third topic through the one wire. The - # storage key has no (bu, agent) because a user table has no Odoo scope; per-user by the - # table's own wall (creator or admin, `user_tables.may_open`). - if scope.startswith("ut_"): - from routes_tables import ut_assembly - - storage_key = f"{session.tenant}:{scope}:{session.uname}" - try: - # ⭐⭐ W31-T20 / D-174 — `with_rows=False`. THIS ROUTE RENDERS NO ROW: the grid fetches - # them from `/tables/{key}/rows` or `/odoo-tables/{key}/rows` in the same paint, and - # `useCustomerData.load` fires both calls in one `Promise.all`. Building the table here - # bought nothing and cost everything — it is why `?scope=ut_odoo_gl_lines` answered - # `409 window_required` six times out of six on the live deploy while the ROWS door for - # the same table served a page in ~150 ms, i.e. a grid that could not be opened at all. - g = ut_assembly(session, scope, storage_key=storage_key, with_rows=False) - except grid_events.StoreUnavailable: - raise err(503, "store_unavailable", "the tenant store is unavailable") - workspace = g["workspace"] - workspace["overlays"] = g["ws"].get("overlays") or {} - # The field contract rides the cheap re-read on EVERY topic — see the customer branch - # below for why (a user table's first cohort changes its contract too: `_cohorts(ctx)` - # is scope-parameterized, so `ut_` surfaces have their own sets). - workspace["fields"] = g["fields"] - workspace["measures"] = g["measures"] - workspace["measureSets"] = g["measure_sets"] - # ⚠ `g["derived"]`, NOT `{}` (2026-08-04). R9 gave every topic its own cohort sets, and - # `ut_assembly` has been building this topic's cohort CELLS since — but this route threw - # them away, so the Locked-views column arrived on the cheap re-read with permanently - # empty values. A column that exists and can never have one is worse than an absent - # column: it reads as "this record is in no locked view", which is a claim. - workspace["derived"] = {str(pid): cells for pid, cells in (g["derived"] or {}).items()} - workspace["viewer"] = {"name": session.uname, "isAdmin": bool(session.admin)} - try: - from core import users as _users - workspace["userOptions"] = _users.assignable_people(tenant=session.tenant) - workspace["userAvatars"] = {k: v for k, v in - _users.avatar_map(tenant=session.tenant).items() - if str(v).startswith("data:image/")} - except Exception: - workspace["userOptions"] = [] - workspace["userAvatars"] = {} - workspace["scopeKey"] = scope - # ⭐ W31-T20 — R6's SECOND SENTENCE REACHES THE BROWSER. On a read-through grid too large - # for one window the pid set is empty, so cohort membership and a shared view's - # `memberPids` are UNRESOLVED here (both fail closed, and `workspace_wire` already stamps - # `missing` on a shortened cohort). An unannounced empty scope is the silent limit R6 - # forbids; this is the announcement, and W31-T22 is the ticket that renders it. - workspace["limits"] = g.get("limits") or [] - return {"workspace": workspace} - - # ── Wave 16 C-TOPIC: the PRODUCT surface has its own assembly (own pool, own field - # contract, own workspace BUCKET) and the contract's own storage key. It branches before - # the customer derivation because the customer storage key embeds (bu, agent) while the - # product one is deliberately `all:all` — one product workspace per user (the wave doc's - # C-TOPIC line), since BU narrows the product VALUES, not which workspace you own. - if scope == "product": - from routes_products import MODULE as PRODUCT_MODULE, product_assembly - - # ⛔ THE GATE CHANGES WITH THE SCOPE (wave 21 C4: the dependency is session-only now, - # so this line IS the product wall — not a second one on top of customer_data). - session.require(PRODUCT_MODULE) - storage_key = f"{session.tenant}:product-list:{session.uname}:all:all" - try: - g = product_assembly(session, scope=scope, storage_key=storage_key) - except grid_events.StoreUnavailable: - raise err(503, "store_unavailable", "the tenant store is unavailable") - workspace = g["workspace"] - workspace["overlays"] = g["ws"].get("overlays") or {} - # R9 made cohorts per-topic, so the PRODUCT surface has its own sets and its own - # first-cohort contract change. Same reason as the customer branch below. - workspace["fields"] = g["fields"] - # The measure channel is EMPTY on this topic (customer-grain descope) — stated - # explicitly so the client's pickers grey rather than guess. - workspace["measures"] = g["measures"] - workspace["measureSets"] = g["measure_sets"] - # `g["derived"]` — the same correction as the user-table branch above, for the same - # reason: `product_assembly` builds this topic's cohort cells and this route discarded - # them. The measure half stays empty on this topic by descope, which is a different - # statement and one the offer already makes. - workspace["derived"] = {str(pid): cells for pid, cells in (g["derived"] or {}).items()} - workspace["viewer"] = {"name": session.uname, "isAdmin": bool(session.admin)} - try: - from core import users as _users - workspace["userOptions"] = _users.assignable_people(tenant=session.tenant) - workspace["userAvatars"] = {k: v for k, v in - _users.avatar_map(tenant=session.tenant).items() - if str(v).startswith("data:image/")} - except Exception: - workspace["userOptions"] = [] - workspace["userAvatars"] = {} - workspace["scopeKey"] = scope - return {"workspace": workspace} - - # The storage key mirrors the host's own convention byte-for-byte (app.py's `_table_grid` - # call sites) with the session's TENANT in the host's hardcoded slot, so localStorage state - # carries across embed ⇄ standalone on the same browser. - # Wave 15 R1 — the SAME derivation the pool uses (`routes_customers._team_agent`), so the - # storage key cannot drift from the scope it names. Value-identical for every migrated - # record (verify_perm_scope section D proves the derivation reproduces the legacy scope - # exactly), so nobody's saved localStorage state moves when the migration runs. - import core.perm_scope as perm_scope - team_id, agent = perm_scope.derive_pool_scope(session.user, MODULE) - bu = team_id if team_id is not None else "all" - if scope == "cohort": - storage_key = f"{session.tenant}:cohort:{session.uname}:{bu}" - else: - storage_key = f"{session.tenant}:customer-list:{session.uname}:{bu}:{agent or 'all'}" - - # ⛔ THE WIRE SHAPE IS THE SHARED PROJECTION (`aios_grid.workspace_wire`), not the store - # shape. The first version returned the store dict with no `storageKey` — and the client - # validator requires one, so the standalone shell DISCARDED the whole workspace: saved views - # never rendered, `cohortMode` never arrived, and the Cohort route silently drew the Customer - # surface. One projection for both servers is the fix that cannot drift. - try: - g = grid_assembly(session, scope=scope, storage_key=storage_key) - except grid_events.StoreUnavailable: - raise err(503, "store_unavailable", "the tenant store is unavailable") - workspace = g["workspace"] - # ADDITIVE to the wire: the caller's own overlay cells. The grid reads overlays from the - # /customers rows, but this route is the cheap read-back a write can be verified against - # (verify_api's overlay probe) without paying the pool call. Per-user by construction — - # `table_workspace` is this session's workspace. - workspace["overlays"] = g["ws"].get("overlays") or {} - # ⭐ THE FIELD CONTRACT, 2026-08-04 — and it is NOT decoration. - # - # `fields_from_workspace(ws, cohorts=bool(cohort_lists))` appends the derived "Locked - # views" column ONLY when the caller owns at least one cohort. So a user's FIRST cohort - # CHANGES THE FIELD CONTRACT — and the rows call that used to be the only carrier of - # `fields` is deliberately never re-fetched on a write (it is a 15-minute-cached Odoo - # pull; this route is the cheap re-read). The client therefore had no way to learn about - # that column short of a remount, which is the second half of the owner's "it only shows - # up when I switch modules and come back". - # - # ⚠ `g["fields"]` is the SAME list `_payload` serves on `/customers` — same assembly, - # same permission wall (`hidden_keys` already applied) — so the two wires cannot disagree - # about what a column is. Taking it from anywhere else would fork the contract that - # verify_fields_contract.py exists to keep single-sourced. - workspace["fields"] = g["fields"] - # ── the STANDALONE measure channel (owner item 1, 2026-07-31) ──────────────────────────── - # The embed receives these as top-level render args; standalone lifts them off THIS route - # (useCustomerData merges them into the payload slots the grid already reads). They ride - # the workspace rather than /customers because a durable write re-reads exactly this route - # (WORKSPACE_STALE), so a new measure column populates without refetching the heavy pool. - workspace["measures"] = g["measures"] - workspace["measureSets"] = g["measure_sets"] - # Derived cells (cohort column + measure columns), keyed by pid. JSON object keys are - # strings; the client indexes with String(pid). - workspace["derived"] = {str(pid): cells for pid, cells in g["derived"].items()} - # Who is looking (permissions verdicts) + the assignee choices for `user`-typed columns — - # the two other host-only render args the shell was missing (fail-closed without them: - # restricted fields uneditable, assignee picker empty). - workspace["viewer"] = {"name": session.uname, "isAdmin": bool(session.admin)} - try: - from core import users as _users - workspace["userOptions"] = _users.assignable_people(tenant=session.tenant) - # Wave 14 C-AVATAR — the options vocabulary's companion: display name -> data URL. - # Absent entries fall back to the client's initials disc; a non-data: value is - # dropped (defence in depth beside the write-side wall in routes_auth). - workspace["userAvatars"] = {k: v for k, v in - _users.avatar_map(tenant=session.tenant).items() - if str(v).startswith("data:image/")} - except Exception: - workspace["userOptions"] = [] - workspace["userAvatars"] = {} - - # THE SURFACE STAMP — a MIRROR of the host's own (`_table_grid`'s hide/cohort stamps). - # ⚠ Deliberately NOT `hideViews`: `cohortMode` is what swaps the Views panel for the cohort - # list, and two switches for one behaviour would drift. - workspace["scopeKey"] = scope - if scope == "cohort": - workspace["cohortMode"] = True - # The create pane offers "this cohort table only" (default) vs "all customer tables". - workspace["scopeChoice"] = True - return {"workspace": workspace} - - -# ─────────────────────────────────────────────── the TIME-SERIES channel (C-TS, 2026-08-02) -_TS_BUCKETS = ("week", "month", "quarter", "year") -_TS_MAX_BUCKETS = 120 -_TS_MAX_FIELDS = 12 -_TS_MAX_PIDS = 5000 -_TS_MAX_LAST_N = 120 -#: C-TSWIN (wave 14): sliding windows cost ONE aggregate query per (metric, bucket) — the -#: price of "bucket N == the grid cell at bucket N's end". The product cap keeps a legal -#: 120-bucket × 12-metric ask from becoming 1,440 queries in one request; typical panels -#: (12 × 3) sit two orders of magnitude under it. Dated amendment in the split doc. -_TS_MAX_CELLS = 720 - - -def _ts_start_of(bucket, d): - """The calendar START of the bucket holding `d` (week = Monday, the vocabulary rule).""" - if bucket == "week": - return d - dt.timedelta(days=d.weekday()) - if bucket == "month": - return d.replace(day=1) - if bucket == "quarter": - return d.replace(month=((d.month - 1) // 3) * 3 + 1, day=1) - return d.replace(month=1, day=1) - - -def _ts_next(bucket, d): - if bucket == "week": - return d + dt.timedelta(days=7) - if bucket == "year": - return d.replace(year=d.year + 1) - step = 1 if bucket == "month" else 3 - m = d.month + step - return dt.date(d.year + (m - 1) // 12, (m - 1) % 12 + 1, 1) - - -def _ts_prev(bucket, d): - if bucket == "week": - return d - dt.timedelta(days=7) - if bucket == "year": - return d.replace(year=d.year - 1) - step = 1 if bucket == "month" else 3 - y, m = d.year, d.month - step - while m < 1: - m += 12 - y -= 1 - return dt.date(y, m, 1) - - -def _ts_label(bucket, start): - if bucket == "week": - return f"Wk of {start.strftime('%b %d')}" - if bucket == "month": - return start.strftime("%b %Y") - if bucket == "quarter": - return f"Q{(start.month - 1) // 3 + 1} {start.year}" - return str(start.year) - - -@router.post("/grid/timeseries") -def grid_timeseries(body: dict = Body(default=None), scope: str = "customer", - session: Session = Depends(module_gate(MODULE))): - """Pooled measure values per calendar bucket — the time-series view's data channel. - - C-TSWIN (wave 14, ruling R1): AS-OF semantics. Each metric's OWN stored window is - re-resolved per bucket with `today := min(bucket end, real today)` — `ytd` is cumulative - from Jan 1, `last_90_days` trailing, `all_time` cumulative ever; bucket N equals what the - grid's measure column would show if today were bucket N's end. Fixed-range (`custom`) - windows cannot slide and are dropped per field as `window_fixed`. Rows carry - `window: {kind, label}` so a cumulative row cannot be misread as periodic, and there is - NO `total` column — sliding windows overlap, so a sum of columns would double-count. - - The client sends the pids its view currently matches (the filter IS the scope); the server - intersects them with the session's book, so the request can only ever NARROW. Values are - POOLED aggregates computed by the semantic layer's own expression (the additivity law: an - average is computed at pool grain, never averaged over per-customer answers). A bucket - with no rows is 0 for a sum/count and null for anything else; a bucket that has not - STARTED yet is null for every kind — an unstarted period's YTD is unanswered, not 0. - """ - from core import measure_resolve - from harness import windows as _wn - from routes_customers import _pool_stamp, _team_agent, allowed_pids - - # Wave 16 C-TOPIC: the channel is CUSTOMER-GRAIN (measure_resolve's own grain), so the - # product surface is refused in words rather than answered with an id-space accident — - # product pids are CRC32 hashes and would mostly fall outside the customer book anyway, - # but "mostly" is not a wall. The client hides the mode for this topic; this is the - # server's half of the same refusal. - _sc = _scope_or_400(scope) - if _sc == "product" or _sc.startswith("ut_"): - raise err(400, "bad_timeseries", - "this surface has no time-series channel — measures are customer-grain") - - body = body or {} - bucket = body.get("bucket") - if bucket not in _TS_BUCKETS: - raise err(400, "bad_timeseries", "bucket must be one of week, month, quarter, year") - raw_fields = body.get("fields") - if not isinstance(raw_fields, list) or not raw_fields: - raise err(400, "bad_timeseries", "fields must be a non-empty list of field keys") - if len(raw_fields) > _TS_MAX_FIELDS: - raise err(400, "bad_timeseries", f"at most {_TS_MAX_FIELDS} fields per request") - raw_pids = body.get("pids") - if not isinstance(raw_pids, list) or not raw_pids: - raise err(400, "bad_timeseries", "pids must be a non-empty list") - if len(raw_pids) > _TS_MAX_PIDS: - raise err(400, "bad_timeseries", f"at most {_TS_MAX_PIDS} pids per request") - try: - wanted = {int(p) for p in raw_pids} - except (TypeError, ValueError): - raise err(400, "bad_timeseries", "pids must be integers") - pool = wanted & {int(p) for p in allowed_pids(session)} - if not pool: - # 403, not an empty 200: the caller asked about customers outside their book, and an - # all-zero series over nobody would read as "no activity", not "not yours". - raise err(403, "out_of_scope", "none of those customers are in your book") - - span = body.get("span") - if not isinstance(span, dict): - raise err(400, "bad_timeseries", "span must be {'lastN': n} or {'from': .., 'to': ..}") - today = time.strftime("%Y-%m-%d") - t = dt.date.fromisoformat(today) - n = span.get("lastN") - starts = [] - if n is not None: - if not isinstance(n, int) or isinstance(n, bool) or not (1 <= n <= _TS_MAX_LAST_N): - raise err(400, "bad_timeseries", f"lastN must be 1..{_TS_MAX_LAST_N}") - cur = _ts_start_of(bucket, t) - starts = [cur] - for _ in range(n - 1): - cur = _ts_prev(bucket, cur) - starts.append(cur) - starts.reverse() - else: - try: - d_from = dt.date.fromisoformat(str(span.get("from"))) - d_to = dt.date.fromisoformat(str(span.get("to"))) - except (TypeError, ValueError): - raise err(400, "bad_timeseries", "span.from/to must be ISO dates (YYYY-MM-DD)") - if d_from > d_to: - d_from, d_to = d_to, d_from - cur = _ts_start_of(bucket, d_from) - while cur <= d_to: - starts.append(cur) - if len(starts) > _TS_MAX_BUCKETS: - raise err(400, "bad_timeseries", - f"that span is more than {_TS_MAX_BUCKETS} {bucket} buckets - " - f"narrow it") - cur = _ts_next(bucket, cur) - if not starts: - raise err(400, "bad_timeseries", "the span holds no buckets") - ends = [_ts_next(bucket, s) - dt.timedelta(days=1) for s in starts] - - rt = session.runtime - if not rt.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable") - import modules.customer_data as cl_mod - # Read-only route: it must not consume a one-shot label-correction ack the browser has - # not seen yet (the same rule event validation follows). - ws = cl_mod.table_workspace(session.uname, consume_corrections=False) - fdefs = ws.get("fields") or {} - keys, dropped, seen = [], [], set() - for k in raw_fields: - k = str(k or "") - if not k or k in seen: - continue - seen.add(k) - fd = fdefs.get(k) - if not isinstance(fd, dict) or not isinstance(fd.get("measure"), dict): - # Named, never silent: v1 eligibility is measure-backed fields only (C-TS). - dropped.append({"field": k, "reason": "not_a_measure_field"}) - continue - keys.append(k) - # ⚡ WAVE 17 R9 (amendment 2026-08-03, GRID's cross-fence ask) — A SHEET WITH NO MEASURE - # FIELDS IS NO LONGER A 400. It answers with the BUCKET GRID and no rows. - # - # Why this is the honest direction and not a loosening: the bucket starts and their labels - # are SERVER math (`_ts_start_of` / `_ts_next` / `_ts_label`), and R9 has the client - # synthesizing snapshot rows for preset columns that have no window to slide. Those rows - # must be painted under the SAME headings as everything else, so the client needs the - # columns even when the server has no series to put in them. Refusing the whole request - # meant the panel could offer nothing at all on this tenant — the shipped contract has zero - # measure-backed presets (`aios_grid.py`: "this branch is currently MEMBERLESS"), which is - # what item 5 is actually about. - # - # Nothing is invented by this: `rows` is empty and `meta.dropped` NAMES every field and why. - # A 400 is still returned for a request that is malformed (bad bucket, bad span, no fields - # at all) — this only stops treating "I asked about columns you cannot serve" as an error. - mfields, slid_keys = [], [] - for k in keys: - m = dict(fdefs[k]["measure"]) - if (m.get("window") or {}).get("kind") == "custom": - # C-TSWIN: a fixed date range cannot slide across buckets — named, never silent, - # the same posture as not_a_measure_field. - dropped.append({"field": k, "reason": "window_fixed"}) - continue - mfields.append({"key": k, "measure": m}) - slid_keys.append(k) - # Same amendment as above: every metric being fixed-range is a sheet with no SERIES, not a - # broken request. The columns still stand, and `window_fixed` still names each refusal. - if len(starts) * len(mfields) > _TS_MAX_CELLS: - raise err(400, "bad_timeseries", - "that ask is too wide - narrow the span or pick fewer metrics") - - team_id, agent = _team_agent(session) - stamp = _pool_stamp(rt, team_id, agent) - problems = [] - bucket_bounds = [(s.isoformat(), e.isoformat()) for s, e in zip(starts, ends)] - # No slidable metrics = nothing to resolve. Skipping the call rather than asking the - # resolver about an empty list keeps the memo free of a meaningless key. - answers = measure_resolve.series_values( - mfields, bucket, bucket_bounds, - team_id, frozenset(pool), today, stamp, rt.series_memo, - on_error=lambda tag, e: problems.append(str(e)[:200])) if mfields else {} - - columns = [] - for s, e in zip(starts, ends): - col = {"key": s.isoformat(), "label": _ts_label(bucket, s), - "from": s.isoformat(), "to": e.isoformat()} - if e > t: - col["partial"] = True - columns.append(col) - rows = [] - for k in slid_keys: - ans = answers.get(k) - if ans is None: - dropped.append({"field": k, "reason": "unresolvable"}) - continue - vals_by = ans.get("values") or {} - agg_kind = str(ans.get("agg") or "sum") - zero_fill = agg_kind in ("sum", "count") - vals = [] - for s in starts: - if s > t: - # C-TSWIN: an unstarted period is unanswered for EVERY agg kind — a future - # month's YTD zero-filled to 0 would read as "the year reset". - vals.append(None) - else: - vals.append(vals_by.get(s.isoformat(), 0 if zero_fill else None)) - wspec = (fdefs[k].get("measure") or {}).get("window") - wnorm = _wn.normalize(wspec) or {} - rows.append({"field": k, "label": str(fdefs[k].get("label") or k)[:120], - "agg": agg_kind, "values": vals, - "window": {"kind": str(wnorm.get("kind") or ""), - "label": _wn.label(wspec)}}) - meta = {"pool": len(pool), "today": today, "bucket": bucket} - if dropped: - meta["dropped"] = dropped - if problems: - meta["problems"] = problems[:5] - return {"columns": columns, "rows": rows, "meta": meta} - - -#: C-CAL caps (wave 17, owner item 9 / ruling R4). A month of days, a handful of metrics. -#: Every one of these is a 400 WITH ITS REASON, never a silent trim: a calendar that quietly -#: answered 20 of the 31 days asked about would paint eleven blank cells that look like days -#: with no activity. -_CAL_MAX_GROUPS = 31 -_CAL_MAX_FIELDS = 6 -_CAL_MAX_CELLS = 186 - - -@router.post("/grid/calendar_metrics") -def grid_calendar_metrics(body: dict = Body(default=None), scope: str = "customer", - session: Session = Depends(module_gate(MODULE))): - """Per-DAY measure values with each metric's own window slid to that day (C-CAL / R4). - - ⛔ WHY THIS EXISTS AT ALL. The calendar's summary cells used to aggregate the row VALUES the - grid already held — which for a measure column means "every member's YTD **as of today**", - summed and printed under a date in March. The number was arithmetically fine and semantically - a lie: it answered a question about today while sitting in a cell labelled with another day. - R4: a metric in a day cell is computed AS OF THAT DAY. - - The shape differs from the time-series channel in exactly one way, and it is the reason this - is a separate route rather than a parameter: **every group carries its OWN pid set**. A - calendar day holds the records the date field placed there, so day-to-day the subject - changes. One `allowed_pids` for the whole request — the TS channel's shape — would compute - each day over everybody, which is a different question again. - - Static (non-measure) fields are NOT served here. They have no window to slide, so the - client's own per-day aggregation over row values stays correct for them; sending them would - invite a second implementation of arithmetic that already works. - """ - from core import measure_resolve - from harness import windows as _wn - from routes_customers import _pool_stamp, _team_agent, allowed_pids - - # The same refusal the TS channel makes, for the same reason: measures are customer-grain - # and product pids are CRC32 hashes of SKU codes. "Mostly outside the book" is not a wall. - _sc_cal = _scope_or_400(scope) - if _sc_cal == "product" or _sc_cal.startswith("ut_"): - raise err(400, "bad_calendar_metrics", - "the product surface has no measure channel yet — measures are customer-grain") - - body = body or {} - raw_groups = body.get("groups") - if not isinstance(raw_groups, list) or not raw_groups: - raise err(400, "bad_calendar_metrics", - "groups must be a non-empty list of {key: 'YYYY-MM-DD', pids: [...]}") - if len(raw_groups) > _CAL_MAX_GROUPS: - raise err(400, "bad_calendar_metrics", - f"at most {_CAL_MAX_GROUPS} days per request (one month)") - raw_fields = body.get("fields") - if not isinstance(raw_fields, list) or not raw_fields: - raise err(400, "bad_calendar_metrics", "fields must be a non-empty list of field keys") - if len(raw_fields) > _CAL_MAX_FIELDS: - raise err(400, "bad_calendar_metrics", f"at most {_CAL_MAX_FIELDS} metrics per request") - if len(raw_groups) * len(raw_fields) > _CAL_MAX_CELLS: - raise err(400, "bad_calendar_metrics", - "that ask is too wide - fewer days or fewer metrics") - - book = {int(p) for p in allowed_pids(session)} - groups, seen_days = [], set() - for g in raw_groups: - if not isinstance(g, dict): - raise err(400, "bad_calendar_metrics", "every group must be an object") - day = str(g.get("key") or "") - try: - d = dt.date.fromisoformat(day) - except (TypeError, ValueError): - raise err(400, "bad_calendar_metrics", - "every group key must be an ISO date (YYYY-MM-DD)") - if day in seen_days: - raise err(400, "bad_calendar_metrics", f"day {day} appears twice") - seen_days.add(day) - raw_pids = g.get("pids") - if not isinstance(raw_pids, list): - raise err(400, "bad_calendar_metrics", "every group needs a pids list") - try: - wanted = {int(p) for p in raw_pids} - except (TypeError, ValueError): - raise err(400, "bad_calendar_metrics", "pids must be integers") - # NARROW-ONLY, per group. The client sends what its calendar placed; the server can - # only ever remove from that, never add. - groups.append((day, d, frozenset(wanted & book))) - if sum(len(p) for _, _, p in groups) > _TS_MAX_PIDS: - raise err(400, "bad_calendar_metrics", - f"at most {_TS_MAX_PIDS} customer references per request") - - rt = session.runtime - if not rt.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable") - import modules.customer_data as cl_mod - ws = cl_mod.table_workspace(session.uname, consume_corrections=False) - fdefs = ws.get("fields") or {} - keys, dropped, seen = [], [], set() - for k in raw_fields: - k = str(k or "") - if not k or k in seen: - continue - seen.add(k) - fd = fdefs.get(k) - if not isinstance(fd, dict) or not isinstance(fd.get("measure"), dict): - # The TS channel's vocabulary, deliberately reused rather than re-coined: the client - # already knows how to say these three words to a reader. - dropped.append({"field": k, "reason": "not_a_measure_field"}) - continue - if ((fd["measure"].get("window") or {}).get("kind") == "custom"): - dropped.append({"field": k, "reason": "window_fixed"}) - continue - keys.append(k) - if not keys: - raise err(400, "bad_calendar_metrics", - "none of the requested fields are measure fields with a window that can " - "slide to a day") - - today = time.strftime("%Y-%m-%d") - t = dt.date.fromisoformat(today) - team_id, agent = _team_agent(session) - stamp = _pool_stamp(rt, team_id, agent) - problems = [] - values = {k: {} for k in keys} - for day, d, pids in groups: - # A day that has not happened is unanswered for EVERY aggregate kind — the unstarted - # bucket law at day grain. Answering 0 would say "we sold nothing", which is a claim - # about a day nobody has lived through yet. - if d > t or not pids: - for k in keys: - values[k][day] = None - continue - answers = measure_resolve.series_values( - [{"key": k, "measure": dict(fdefs[k]["measure"])} for k in keys], - "day", [(day, day)], team_id, pids, today, stamp, rt.series_memo, - on_error=lambda tag, e: problems.append(str(e)[:200])) - for k in keys: - ans = answers.get(k) - if ans is None: - values[k][day] = None - continue - vals_by = ans.get("values") or {} - zero_fill = str(ans.get("agg") or "sum") in ("sum", "count") - values[k][day] = vals_by.get(day, 0 if zero_fill else None) - - for k in keys: - if all(v is None for v in values[k].values()): - # Every day unanswerable is a FIELD-level failure, and saying so is the difference - # between "no activity that month" and "this metric could not be computed". - dropped.append({"field": k, "reason": "unresolvable"}) - out = {"values": values, "today": today, - "windows": {k: {"kind": str((_wn.normalize( - (fdefs[k].get("measure") or {}).get("window")) or {}).get("kind") or ""), - "label": _wn.label((fdefs[k].get("measure") or {}).get("window"))} - for k in keys}} - if dropped: - out["dropped"] = dropped - if problems: - out["problems"] = problems[:5] - return out - - -#: ⭐⭐ THE BULK CELL DOOR'S OWN CEILING, and it is REPORTED rather than silently enforced. The -#: standing no-cap rule governs data read from a connected source; this is a WRITE of team-typed -#: values, so a bound is right — but R6's second sentence still binds, which is why exceeding it -#: is a 400 naming the number and the count, never a truncation. -#: Sized against the real job with headroom: the 2027 catalog is 1,397 rows. -MAX_BULK_ROWS = 10_000 - - -@router.post("/grid/bulk-cells") -def bulk_cells(body: dict = Body(default=None), - session: Session = Depends(require_session)): - """`{scopeKey, rows: {"": {key: value}}}` → shared cells on many rows, ONE transaction. - - ⭐⭐ WHY THIS EXISTS AT ALL, because "we already have `/grid/events`" is the obvious objection. - That door is capped at `_MAX_EVENTS = 24` (it is sized for the client's resend window), so a - 1,397-row import is ~59 sequential POSTs against ONE JSON document — and this repo has a - MEASURED scar for exactly that: 18 writes against one document under the store's coalescing - single-flight landed **zero** while answering 200 eighteen times. Raising `_MAX_EVENTS` would - have widened the browser's own resend window to fix a script's problem. One door, one - transaction, is the honest shape. - - ⛔ TENANT-WIDE KEYS ONLY, AND THAT IS THE POINT RATHER THAN A LIMITATION. It writes through - `shared_overlay`, which every account in the tenant reads. A per-user bulk write would be a - contradiction: nobody imports 1,397 rows of the team's work so that one account can see it — - that is precisely the failure owner ruling R6 was made to avoid. A key that is not declared - `shared: true` in the canonical contract is REFUSED BY NAME. - - ⛔ ADMIN ONLY. This mutates data every account in the tenant reads, in bulk, in one call. - D-172 already records that the shared stratum has an asymmetric wall (anyone may create a - tenant-wide column, only the creator or an admin may delete one); a new door does not get to - inherit the loose half of an asymmetry somebody already flagged. - """ - import core.shared_overlay as shared_overlay - import modules.product_data as pd - import core.perm_scope as perm_scope - from routes_products import MODULE as PRODUCT_MODULE, scoped_pool - - scope = _scope_or_400((body or {}).get("scopeKey")) - if scope != "product": - # Fail-closed with the reason: the shared stratum is a PRODUCT-topic mechanism today. - raise err(400, "scope_not_bulk_writable", - f"bulk cell writes are available on the product database; '{scope}' has no " - f"tenant-wide cell stratum") - session.require(PRODUCT_MODULE) - if not session.admin: - raise err(403, "admin_only", - "a bulk write changes values every account in this workspace reads") - - rows = (body or {}).get("rows") - if not isinstance(rows, dict): - raise err(400, "bad_rows", "expected {rows: {\"\": {field: value}}}") - if len(rows) > MAX_BULK_ROWS: - raise err(400, "too_many_rows", - f"at most {MAX_BULK_ROWS} rows per request; this one carried {len(rows)}") - - pids, _team, _src, fields_base = scoped_pool(session) - allowed = {int(p) for p in pids} - shared_keys = set(pd.SHARED_KEYS()) - hidden = perm_scope.hidden_keys(session.user, PRODUCT_MODULE, fields_base) - - clean, unknown_pid, refused_keys = {}, [], {} - for raw_pid, values in rows.items(): - try: - pid = int(raw_pid) - except (TypeError, ValueError): - unknown_pid.append(str(raw_pid)[:40]) - continue - # ⚠ THE POOL IS THE WALL. `scoped_pool` is the same predicate the read door uses, so a - # caller cannot write a row they could not see — including a row in another BU. - if pid not in allowed: - unknown_pid.append(str(raw_pid)[:40]) - continue - keep = {} - for key, value in dict(values or {}).items(): - key = str(key) - if key not in shared_keys: - refused_keys.setdefault("not_shared", set()).add(key) - continue - if key in hidden: - refused_keys.setdefault("hidden_by_permissions", set()).add(key) - continue - if not isinstance(value, (str, int, float)) or isinstance(value, bool): - refused_keys.setdefault("unsupported_value", set()).add(key) - continue - keep[key] = value - if keep: - clean[pid] = keep - - written = shared_overlay.put_rows(pd.TABLE_KEY, clean, st=pd.TABLE_OPS.st) if clean else {} - out = { - "rows_written": len(written), - "cells_written": sum(len(v) for v in written.values()), - "rows_requested": len(rows), - } - # ⭐ R6's second sentence: what did NOT land, and why. A bulk door that reports only its - # successes is how 701 unmatched SKUs disappear quietly. - if unknown_pid: - out["rows_not_in_your_pool"] = {"count": len(unknown_pid), - "sample": sorted(unknown_pid)[:10]} - if refused_keys: - out["refused_fields"] = {reason: sorted(keys) - for reason, keys in refused_keys.items()} - return out - - -@router.post("/grid/events") -def grid_events_route(body: dict = Body(default=None), - session: Session = Depends(require_session)): - """`{events: []}` → `{results, doc?, toast?}`. - - Wave 21 C4: session-only dependency, per-scope gate below — the write door must admit the - same sessions the read door (`/workspace`) admits, or a tenant without the customer module - can SEE its own user tables and not write to them.""" - from core import grid_events - from routes_customers import grid_assembly - - events = (body or {}).get("events") - if events is None and isinstance(body, dict) and body.get("type"): - events = [body] # a single event object, the legacy shape - if not isinstance(events, list): - raise err(400, "bad_events", "expected {events: [...]}") - if len(events) > _MAX_EVENTS: - raise err(400, "too_many_events", - f"at most {_MAX_EVENTS} events per request (the client's resend window)") - - # Validated with the SAME predicate as the read route. An unrecognised scopeKey used to be - # passed through verbatim, and `core.grid_events` only ever compares it to 'cohort' — so a - # typo degraded silently to customer-scope behaviour on a WRITE. Read and write must agree on - # what a scope is, or the surface you read is not the surface you wrote. - scope = _scope_or_400((body or {}).get("scopeKey")) - # Wave 21 C4 — same per-scope gate as /workspace (read and write doors must agree). - if not (scope == "product" or scope.startswith("ut_")): - session.require(MODULE) - # This assembly validates the write; it is not a payload the browser will render. Leave - # one-shot field-name correction acks queued for the subsequent /workspace refresh. - # Wave 16 C-TOPIC: the PRODUCT topic gets the product assembly — product field contract, - # product pids, and (below) the product TABLE OPS, so a product event is validated against - # and lands in the product bucket. The measure/cohort context is honestly EMPTY there: - # `clean_measure_field` refuses measure creates on this surface by construction (the - # customer-grain descope), which is the fail-closed shape, not an accident. - if scope == "product": - from routes_products import MODULE as PRODUCT_MODULE, product_assembly - - session.require(PRODUCT_MODULE) # the product wall (dependency is session-only, C4) - g = product_assembly(session, consume_corrections=False) - elif scope.startswith("ut_"): - # Wave 18 C3-UT — the user-table wall (creator/admin) is inside the assembly; the - # measure/cohort context is honestly EMPTY (customer-grain machinery, no meaning here). - # ⭐⭐ WAVE 30 / W30-T30 (owner item 2: *"when I click hide fields it crash… no matter the - # size"*). This used to be `ut_assembly(...)`, which builds the whole table to validate a - # write it then throws away: `scoped_pool` allocates a dict per row and sorts them, so ONE - # hide-fields checkbox rebuilt ~33k order rows before the event was even dispatched. - # ⛔ NOTHING IS VALIDATED LESS. The comment above still holds — this assembly is the - # permission wall and the admission context — and `ut_write_ctx` returns the SAME six keys - # this route reads, with a pid set derived from exactly the row ids `scoped_pool` would - # have kept. What it does not do is materialise the rows nobody here looks at. - from routes_tables import ut_write_ctx - - g = ut_write_ctx(session, scope) - # ⭐⭐ W31-T20 / D-174 — A WRITE THAT NEEDS THE ROW SET REFUSES OUT LOUD, and this is the - # half that is easy to skip because the code already "fails closed" without it. When a - # read-through grid's population exceeds one window the pid set is EMPTY, and every - # pid-bearing handler in `grid_events` then answers `False` — `overlay_patch` and - # `add_to_list` both `return False` for a pid not in `allowed_pids`. On the wire that is - # `rerender: false` and HTTP 200: a write the user watched succeed, that did nothing - # ([[lost-write-looks-like-failed-read]]). Schema-only events — hide a field, save a view, - # rename a column — name no pid and are untouched, which is D-170. - # ⚠ THE TEST IS THE EVENT'S OWN KEYS, not a kind list: a new pid-bearing event type would - # otherwise inherit the silent no-op the day it is added. - if g.get("limits"): - named = [e for e in events - if isinstance(e, dict) and (e.get("pid") is not None or e.get("pids"))] - if named: - lim = g["limits"][0] - raise err(409, "pid_scope_unresolved", - f"this database is served through the connector mirror and its rows " - f"cannot be listed in one window, so a change addressed to particular " - f"records ({len(named)} of {len(events)} here) cannot be admitted — " - f"{lim.get('cause') or 'the row set is unresolved'}. " - f"{lim.get('recommendation') or ''}".strip()) - else: - g = grid_assembly(session, scope=scope, consume_corrections=False) - # ⚠ THE MEASURE CONTEXT IS NOT OPTIONAL (2026-07-31). Without `measure_offer`, - # `clean_measure_field` had an empty admission list and every measure-column create over - # HTTP was silently refused; without `measure_keys`, `clean_filter_tree` stripped every - # measure CONDITION out of a saved view. The embed always passed these; the API adapter - # simply had not been given them — the standalone shell could read measures it could - # never write. - ctx = _ctx(session, g["fields"], g["pids"], scope_key=scope, - measure_keys=frozenset(m["key"] for m in g["measures"]), - resolved_ids=frozenset(g["measure_sets"]), - cohort_ids=frozenset(c["id"] for c in g["lists"]), - measure_offer=tuple(g["measures"]), - visible_views=tuple(g["views"])) - - # Per-event results so the client can tell which of a batch landed — the component's own - # bridge has no response channel at all, so this is strictly more than the embed gets. - results = [] - try: - for one in events: - eid = str(one.get("id") or "") if isinstance(one, dict) else "" - # ⭐⭐ D-291 — WHICH event was refused, not merely THAT something was. The handler - # appends to a shared list, so the refusals belonging to THIS event are exactly the - # ones that appeared across THIS call. Reading the list once after the loop would - # answer "the batch was refused" and leave the caller to guess which member, which is - # the same class of unfalsifiable answer the channel exists to end. - _before = len(ctx.out.refusals) - rerender = grid_events.handle_one(one, ctx) - row = {"id": eid, "rerender": bool(rerender)} - mine = ctx.out.refusals[_before:] - if mine: - # `refused` is the SHAPE a caller branches on; the first reason is the one that - # stopped this write (a handler returns at its first refusal). - row["refused"] = mine[0] - results.append(row) - except grid_events.StoreUnavailable: - raise err(503, "store_unavailable", - "the tenant store is unavailable — none of your changes were saved") - - out = {"results": results, "rerender": any(r["rerender"] for r in results)} - if ctx.out.refusals: - # ⚠ ALSO AT THE TOP LEVEL, because a batch that was wholly refused must not read as a - # batch that wholly landed. A caller that only checks the envelope still learns something - # is wrong, and a caller that walks `results` learns exactly which member. - out["refusals"] = list(ctx.out.refusals) - if ctx.out.doc is not None: - # ⭐ C4 / W30-T27 — `docPayload` IS THE NAME THE CLIENT ALREADY DECLARES. `types.ts` has - # carried `docPayload?: {pid, docId, name, mime, data_b64}` since C5, and `Documents.tsx` - # matches it against the fetch it is waiting on — while this route has been answering - # `doc`, which `apiBridge.ts` deliberately drops. One object, emitted under the name the - # consumer looks for, so F's wiring needs no translation step to get wrong. - # ⚠ `doc` stays for one wave: nothing in the client reads it, but a gate might, and - # removing a key to save six bytes is not worth a red nobody predicted. - out["doc"] = out["docPayload"] = ctx.out.doc - if ctx.out.toast is not None: - out["toast"] = ctx.out.toast - - # ── ⭐ owner item 2 (2026-08-03): THE NEW MEASURE COLUMN'S VALUES, ONE ROUND TRIP SOONER ── - # - # Creating a measure column cost the browser TWO sequential trips before a single number - # appeared: this one to persist the field, then a whole `/workspace` to compute it. The - # second cannot start until the first lands (the resolver reads the PERSISTED field), so the - # wait was structural, not slow code — the owner's "it takes some time for the data to - # populate". The values are computed here instead, immediately after the write, and ride - # this response. - # - # ⚠ IT COSTS NOTHING EXTRA TO COMPUTE. The expensive part is one DuckDB aggregate over the - # book, and `rt.measure_memo` is keyed on (pool stamp, scope, pool, measure, window) — so - # the `/workspace` re-read that still follows HITS the memo instead of doing this work. The - # query happens once either way; only its position moved. - # - # ⚠ NARROW ON PURPOSE. Gated to an actual measure-column write, so an overlay edit or a - # cohort add — the overwhelming majority of events — never pays for a second assembly. - # - # ⚠ AND IT IS A SHORTCUT, NOT A PATH. Any failure is swallowed: `WORKSPACE_STALE` still - # fires from `rerender`, and the re-read still delivers these values exactly as it does - # today. Nothing depends on this having worked. - if out["rerender"] and scope in ("customer", "cohort") and any( - isinstance(e, dict) and e.get("type") == "field_upsert" - and str(((e.get("field") or {}) if isinstance(e.get("field"), dict) else {}) - .get("key") or "").startswith("measure_") - for e in events): - try: - fresh = grid_assembly(session, scope=scope, consume_corrections=False) - out["derived"] = {str(pid): cells for pid, cells in fresh["derived"].items()} - except Exception: - pass - # ⚠ NOTHING IS INVALIDATED HERE, on purpose. The runtime cache holds ONLY the scope-shaped - # Odoo pool (see `routes_customers._pool_rows`), and no event on this route can change an - # Odoo column — Odoo is read-only. Everything an event DOES change (overlays, fields, views, - # folders, cohorts) is re-read from the store on the next request. An earlier version cleared - # `pool_cache` after an `overlay_patch`, which threw away an expensive Odoo pull to refresh - # data that was never in it. - return out - - -# ── CONTRACT C1 (W36-T20): THE REGISTRY TOPICS' ROW READERS ─────────────────────────────────── -# ⭐⭐ R6 — *"EVERY database gets the same permission logic, always."* `core.perm_scope.scoped_table` -# is the ONE door to any database's rows, and it cannot import a topic's pool builder: `core` never -# imports up (`platform/ARCHITECTURE.md`) and these pools are built by `modules/` + `aios_grid` -# behind this layer's per-tenant cache. So the app layer DECLARES its readers, exactly as -# `routes_odoo_tables` declares connected tables to `user_tables.register_connected`. -# -# ⛔ REGISTERED HERE RATHER THAN IN `routes_customers`/`routes_products` because those two files -# are outside wave 36's lane-C fence. The readers themselves are three lines each and call the -# SAME `_pool_for` + `derive_pool_scope` pair those routes call, so there is no second pool and no -# second scope derivation — only a second CALLER of the one that exists. -# -# ⚠ AND THE TOPIC ROUTES STILL HAVE THEIR OWN DOOR TODAY. Contract C1 says `apply_row_scope` + -# `visible_fields` "move behind" `scoped_table`; moving `routes_customers.grid_assembly` and -# `routes_products.scoped_pool` is booked as a PENDING row (mailbox/C.md, C-1) rather than done -# here, because neither file is in this fence. What ships now is the arm the wave is load-bearing -# on — every `ut_*` database, plus E's sandbox — and a topic arm that is REAL rather than stubbed, -# so `scoped_table`'s topic leg is exercised by the product instead of only by a gate. -def _topic_rt(st, module): - """The tenant runtime a topic pool must be built against, or a REPORTED refusal. - - ⛔ A topic pool is per TENANT (`rt.pool_cache`), so `st=None` cannot be resolved to "the - default" without picking a tenant at random — which on this box is tenant #0's PRODUCTION - data. Standing rule 1's second sentence: say why, and say what to do instead. - """ - if st is None: - import core.perm_scope as perm_scope - raise perm_scope.Unresolvable( - subject="rows", effect="unreadable", - cause=f"'{module}' is a registry topic whose pool is built per tenant and no tenant " - f"runtime was passed", - recommendation="pass the session's runtime as `st=`. A topic pool cannot be " - "resolved without knowing which tenant is asking") - return st - - -def _customer_rows(table_key, user, st): - """`(fields, rows)` for the customer topic — the SAME derivation `_team_agent` uses.""" - import aios_grid - import core.perm_scope as perm_scope - from routes_customers import _pool_for - - rt = _topic_rt(st, table_key) - team_id, agent = perm_scope.derive_pool_scope(user, table_key) - return list(aios_grid.FIELDS), _pool_for(rt, team_id, agent) - - -def _product_rows(table_key, user, st): - """`(fields, rows)` for the product topic. `consolidated=` follows the derived scope, so a - BU-pinned reader gets that BU's field contract rather than the consolidated one.""" - import core.perm_scope as perm_scope - from routes_products import _pool_for, pd_fields - - rt = _topic_rt(st, table_key) - team_id, _agent = perm_scope.derive_pool_scope(user, table_key) - return pd_fields(consolidated=team_id is None), _pool_for(rt, team_id) - - -def _register_topic_rows(): - """Declare both topic readers to C1. Called at import; returns the registered key set. - - ⚠ THE KEYS ARE LITERALS AND THE ROUTE IMPORTS ARE INSIDE THE READERS, on purpose: this runs at - module import, and `from routes_products import MODULE` here would pull a sibling router in - before its own imports have settled. Every other cross-router reference in this file is lazy - for the same reason. The literals are held to their sources by `verify_scopes`, so they cannot - drift into naming a topic that does not exist. - """ - import core.perm_scope as perm_scope - - perm_scope.register_rows(_customer_rows, MODULE) - return perm_scope.register_rows(_product_rows, _PRODUCT_MODULE) - - -#: ⚠ `_`-prefixed, because three functions in this file already bind the name `PRODUCT_MODULE` -#: LOCALLY from `routes_products`. A module-level twin of that spelling would read as the same -#: thing and be a different one — [[constant-two-features-share]] waiting to happen. -_PRODUCT_MODULE = "product_data" - -_C1_ROW_SOURCES = _register_topic_rows() +"""routes_grid.py — X2's write seam over HTTP: the SECOND adapter on `core.grid_events` (EXIT-1c). + +`POST /api/v1/grid/events` takes the component's event objects VERBATIM — the same objects the +Streamlit host receives through the component value slot, unchanged — and runs them through the +same `core.grid_events.handle_events` the Streamlit adapter runs. That is the whole point of +EXIT-1a: one implementation of every permission wall, two transports. A route that re-validated +anything here would be a second wall to keep in step, and the two would drift on the first change. + +DEDUP IS PER REQUEST, and that is a deliberate limit, not an oversight. The Streamlit adapter's +`seen_ids` lives in `st.session_state` — genuinely per user session — because the client resends +its recent 24-event window on every emit inside one page session. A stateless API has no such +dict, and inventing a per-user server-side one would be exactly the resident per-tenant state +EXIT-4a exists to remove. So: ids are deduped WITHIN a request body (the resend window's whole +purpose — a batch that repeats an id processes it once), and a genuinely replayed request is +handled by the operations being idempotent. The one event where a replay is observable is +`add_to_list` (it would add the same pids to the same cohort twice — a set union, so the +membership is unchanged, but the toast count repeats). Noted rather than papered over; a +server-side idempotency key belongs with D2's session mirror (C-2). + +STORE DOWN = 503. `fallback_ws` is None on this adapter, so `core.grid_events` raises +`StoreUnavailable` rather than writing to an in-memory workspace no API request could ever read +back. A 200 over a write that evaporated is the failure this rule exists to prevent. +""" +import datetime as dt +import time + +from fastapi import APIRouter, Body, Depends + +from deps import Session, err, module_gate, perms, require_session + +router = APIRouter(prefix="/api/v1") + +MODULE = "customer_data" + +#: The client's own resend window (`app.py`'s `[-24:]`). A body larger than this is not a +#: legitimate client — refuse it rather than doing 500 store writes on one request. +_MAX_EVENTS = 24 + + +def _ctx(session: Session, fields, pids, **kw): + from core import grid_events + import aios_grid + import core.perm_scope as perm_scope + + # Wave 16 C-TOPIC: the ctx is TOPIC-SHAPED. The product scope swaps all three of the + # things a write is validated against — the module the permission wall reads, the canonical + # contract the hidden-field closure runs over, and the TABLE OPS the write lands in. + # Getting any one of them from the other topic is the "validated against the wrong field + # contract" near-miss the wave-15 routes_products header warned about. + scope_key = str(kw.get("scope_key") or "") + if scope_key == "product": + import modules.product_data as pd + from routes_products import MODULE as _PMOD, pd_fields + + module, canonical = _PMOD, pd_fields(consolidated=True) + kw.setdefault("table", pd.TABLE_OPS) + hidden = perm_scope.hidden_keys(session.user, module, canonical) + elif scope_key.startswith("ut_"): + # Wave 18 C3-UT: a user table has no module in the permission wall (its wall is + # `user_tables.may_open`, already applied by the assembly this route ran first), so + # the hidden-field closure is EMPTY rather than borrowed from another topic's contract. + import core.table_store as table_store + + kw.setdefault("table", + table_store.make(f"{scope_key}_table_workspace", st=session.runtime)) + hidden = frozenset() + else: + module, canonical = MODULE, aios_grid.FIELDS + hidden = perm_scope.hidden_keys(session.user, module, canonical) + return grid_events.EventCtx( + uname=session.uname, allowed_pids=pids, fields=fields, admin=session.admin, + # C-PERM: the write wall's field half, on the EVENTS transport too. This route takes the + # component's event objects verbatim, so a hidden key would otherwise arrive here with + # nothing between it and the store. + hidden_keys=hidden, + # ⭐ WAVE 25 (R6b, closes D-16) — THE TENANT HANDLE, on EVERY scope. This is the line + # D-16's exit condition names, and without it the rest of the fix is inert: the handler + # falls back to the module-global `core.store`, which is tenant #0's repo, so a Nurilab + # user's documents, cohorts and — through `_tops` — their whole customer/product table + # workspace were written into Royal Imports' dataset. Note it is set for the CUSTOMER + # and PRODUCT branches too, not only `ut_`: those two never passed a scoped `table`, so + # they were the ones actually resolving to tenant #0 on every request. + st=session.runtime, + fallback_ws=None, seen_ids={}, **kw) + + +#: The surfaces the ONE grid serves. `cohort` is the same table over hand-curated SETS rather +#: than over the whole scoped pool — `app.py:page_cohort` calls the same `_table_grid` with +#: `cohort_mode=True, scope_key='cohort'`. `product` (wave 16 C-TOPIC) is the SKU table: +#: same engine, its own pool, field contract and workspace BUCKET (routes_products). +_SCOPES = ("customer", "cohort", "product") + + +def _scope_or_400(raw): + """⛔ REFUSE AN UNKNOWN SCOPE, never default it. A typo silently served as `customer` would + hand a user the whole book on a page they opened to see one cohort — the widening direction, + which is the one that must fail closed. + + Wave 18 (C3-UT): a `ut_`-prefixed scope names a USER TABLE and passes through here — + existence and the per-table wall are enforced by `routes_tables.ut_assembly` (404/403), + which every consumer of such a scope goes through. Passing an unknown ut key therefore + still fails closed, just one layer down where the store can actually be consulted.""" + scope = (raw or "customer").strip().lower() + if scope.startswith("ut_"): + return scope + if scope not in _SCOPES: + raise err(400, "bad_scope", + f"scope must be one of {', '.join(_SCOPES)} — refusing to guess") + return scope + + +@router.get("/workspace") +def workspace(scope: str = "customer", + session: Session = Depends(require_session)): + """The durable table workspace for this session: views, fields, overlays, folders. + + ⛔ WAVE 21 (C4): the wall is TOPIC-SHAPED, so the DEPENDENCY is session-only and each scope + asserts its own gate below. The old `module_gate("customer_data")` dependency 403'd a + `ut_*` workspace for any tenant whose catalogue omits the customer module (loopable/ + nurilab/gtmlab ship modules w/o it) — and the client then fell back to the shared + localStorage bucket + demo data, which is exactly the "new database shows RI's customer + fields" defect. A user table's real wall is `user_tables.may_open`, enforced inside + `ut_assembly` (404/403, one layer down where the store can be consulted). + + ⚠ WHY THIS EXISTS (S1↔S2, 2026-07-30 — an X2 AMENDMENT, see the split doc). X2 fixes + `/customers` at the exact shape `verify_fields_contract.py` referees, so the workspace cannot + ride along in it without forking the thing that gate keeps single-sourced. Without a + workspace route the standalone shell's write path would be WRITE-ONLY: events persist + server-side and nothing reads them back on reload, so a saved view looks lost to the user + even though the store has it. This route closes that read-back gap at its own URL. + + `allowed_pids` is passed so a SHARED view's `memberPids` are re-scoped to THIS reader — the + wave-9 leak rule. Omitting it would hand a Fisch-scoped user a member list built by a + full-access user. + """ + from core import grid_events + from routes_customers import grid_assembly + + scope = _scope_or_400(scope) + # Wave 21 C4 — the per-scope gate the dependency no longer asserts: customer/cohort need + # the customer module; product asserts PRODUCT_MODULE in its branch; ut_* needs only a + # session (its wall is the table's own, inside the assembly). + if not (scope == "product" or scope.startswith("ut_")): + session.require(MODULE) + # ── Wave 18 C3-UT: a USER TABLE'S workspace — the third topic through the one wire. The + # storage key has no (bu, agent) because a user table has no Odoo scope; per-user by the + # table's own wall (creator or admin, `user_tables.may_open`). + if scope.startswith("ut_"): + from routes_tables import ut_assembly + + storage_key = f"{session.tenant}:{scope}:{session.uname}" + try: + # ⭐⭐ W31-T20 / D-174 — `with_rows=False`. THIS ROUTE RENDERS NO ROW: the grid fetches + # them from `/tables/{key}/rows` or `/odoo-tables/{key}/rows` in the same paint, and + # `useCustomerData.load` fires both calls in one `Promise.all`. Building the table here + # bought nothing and cost everything — it is why `?scope=ut_odoo_gl_lines` answered + # `409 window_required` six times out of six on the live deploy while the ROWS door for + # the same table served a page in ~150 ms, i.e. a grid that could not be opened at all. + g = ut_assembly(session, scope, storage_key=storage_key, with_rows=False) + except grid_events.StoreUnavailable: + raise err(503, "store_unavailable", "the tenant store is unavailable") + workspace = g["workspace"] + workspace["overlays"] = g["ws"].get("overlays") or {} + # The field contract rides the cheap re-read on EVERY topic — see the customer branch + # below for why (a user table's first cohort changes its contract too: `_cohorts(ctx)` + # is scope-parameterized, so `ut_` surfaces have their own sets). + workspace["fields"] = g["fields"] + workspace["measures"] = g["measures"] + workspace["measureSets"] = g["measure_sets"] + # ⚠ `g["derived"]`, NOT `{}` (2026-08-04). R9 gave every topic its own cohort sets, and + # `ut_assembly` has been building this topic's cohort CELLS since — but this route threw + # them away, so the Locked-views column arrived on the cheap re-read with permanently + # empty values. A column that exists and can never have one is worse than an absent + # column: it reads as "this record is in no locked view", which is a claim. + workspace["derived"] = {str(pid): cells for pid, cells in (g["derived"] or {}).items()} + workspace["viewer"] = {"name": session.uname, "isAdmin": bool(session.admin)} + try: + from core import users as _users + workspace["userOptions"] = _users.assignable_people(tenant=session.tenant) + workspace["userAvatars"] = {k: v for k, v in + _users.avatar_map(tenant=session.tenant).items() + if str(v).startswith("data:image/")} + except Exception: + workspace["userOptions"] = [] + workspace["userAvatars"] = {} + workspace["scopeKey"] = scope + # ⭐ W31-T20 — R6's SECOND SENTENCE REACHES THE BROWSER. On a read-through grid too large + # for one window the pid set is empty, so cohort membership and a shared view's + # `memberPids` are UNRESOLVED here (both fail closed, and `workspace_wire` already stamps + # `missing` on a shortened cohort). An unannounced empty scope is the silent limit R6 + # forbids; this is the announcement, and W31-T22 is the ticket that renders it. + workspace["limits"] = g.get("limits") or [] + return {"workspace": workspace} + + # ── Wave 16 C-TOPIC: the PRODUCT surface has its own assembly (own pool, own field + # contract, own workspace BUCKET) and the contract's own storage key. It branches before + # the customer derivation because the customer storage key embeds (bu, agent) while the + # product one is deliberately `all:all` — one product workspace per user (the wave doc's + # C-TOPIC line), since BU narrows the product VALUES, not which workspace you own. + if scope == "product": + from routes_products import MODULE as PRODUCT_MODULE, product_assembly + + # ⛔ THE GATE CHANGES WITH THE SCOPE (wave 21 C4: the dependency is session-only now, + # so this line IS the product wall — not a second one on top of customer_data). + session.require(PRODUCT_MODULE) + storage_key = f"{session.tenant}:product-list:{session.uname}:all:all" + try: + g = product_assembly(session, scope=scope, storage_key=storage_key) + except grid_events.StoreUnavailable: + raise err(503, "store_unavailable", "the tenant store is unavailable") + workspace = g["workspace"] + workspace["overlays"] = g["ws"].get("overlays") or {} + # R9 made cohorts per-topic, so the PRODUCT surface has its own sets and its own + # first-cohort contract change. Same reason as the customer branch below. + workspace["fields"] = g["fields"] + # The measure channel is EMPTY on this topic (customer-grain descope) — stated + # explicitly so the client's pickers grey rather than guess. + workspace["measures"] = g["measures"] + workspace["measureSets"] = g["measure_sets"] + # `g["derived"]` — the same correction as the user-table branch above, for the same + # reason: `product_assembly` builds this topic's cohort cells and this route discarded + # them. The measure half stays empty on this topic by descope, which is a different + # statement and one the offer already makes. + workspace["derived"] = {str(pid): cells for pid, cells in (g["derived"] or {}).items()} + workspace["viewer"] = {"name": session.uname, "isAdmin": bool(session.admin)} + try: + from core import users as _users + workspace["userOptions"] = _users.assignable_people(tenant=session.tenant) + workspace["userAvatars"] = {k: v for k, v in + _users.avatar_map(tenant=session.tenant).items() + if str(v).startswith("data:image/")} + except Exception: + workspace["userOptions"] = [] + workspace["userAvatars"] = {} + workspace["scopeKey"] = scope + return {"workspace": workspace} + + # The storage key mirrors the host's own convention byte-for-byte (app.py's `_table_grid` + # call sites) with the session's TENANT in the host's hardcoded slot, so localStorage state + # carries across embed ⇄ standalone on the same browser. + # Wave 15 R1 — the SAME derivation the pool uses (`routes_customers._team_agent`), so the + # storage key cannot drift from the scope it names. Value-identical for every migrated + # record (verify_perm_scope section D proves the derivation reproduces the legacy scope + # exactly), so nobody's saved localStorage state moves when the migration runs. + import core.perm_scope as perm_scope + team_id, agent = perm_scope.derive_pool_scope(session.user, MODULE) + bu = team_id if team_id is not None else "all" + if scope == "cohort": + storage_key = f"{session.tenant}:cohort:{session.uname}:{bu}" + else: + storage_key = f"{session.tenant}:customer-list:{session.uname}:{bu}:{agent or 'all'}" + + # ⛔ THE WIRE SHAPE IS THE SHARED PROJECTION (`aios_grid.workspace_wire`), not the store + # shape. The first version returned the store dict with no `storageKey` — and the client + # validator requires one, so the standalone shell DISCARDED the whole workspace: saved views + # never rendered, `cohortMode` never arrived, and the Cohort route silently drew the Customer + # surface. One projection for both servers is the fix that cannot drift. + try: + g = grid_assembly(session, scope=scope, storage_key=storage_key) + except grid_events.StoreUnavailable: + raise err(503, "store_unavailable", "the tenant store is unavailable") + workspace = g["workspace"] + # ADDITIVE to the wire: the caller's own overlay cells. The grid reads overlays from the + # /customers rows, but this route is the cheap read-back a write can be verified against + # (verify_api's overlay probe) without paying the pool call. Per-user by construction — + # `table_workspace` is this session's workspace. + workspace["overlays"] = g["ws"].get("overlays") or {} + # ⭐ THE FIELD CONTRACT, 2026-08-04 — and it is NOT decoration. + # + # `fields_from_workspace(ws, cohorts=bool(cohort_lists))` appends the derived "Locked + # views" column ONLY when the caller owns at least one cohort. So a user's FIRST cohort + # CHANGES THE FIELD CONTRACT — and the rows call that used to be the only carrier of + # `fields` is deliberately never re-fetched on a write (it is a 15-minute-cached Odoo + # pull; this route is the cheap re-read). The client therefore had no way to learn about + # that column short of a remount, which is the second half of the owner's "it only shows + # up when I switch modules and come back". + # + # ⚠ `g["fields"]` is the SAME list `_payload` serves on `/customers` — same assembly, + # same permission wall (`hidden_keys` already applied) — so the two wires cannot disagree + # about what a column is. Taking it from anywhere else would fork the contract that + # verify_fields_contract.py exists to keep single-sourced. + workspace["fields"] = g["fields"] + # ── the STANDALONE measure channel (owner item 1, 2026-07-31) ──────────────────────────── + # The embed receives these as top-level render args; standalone lifts them off THIS route + # (useCustomerData merges them into the payload slots the grid already reads). They ride + # the workspace rather than /customers because a durable write re-reads exactly this route + # (WORKSPACE_STALE), so a new measure column populates without refetching the heavy pool. + workspace["measures"] = g["measures"] + workspace["measureSets"] = g["measure_sets"] + # Derived cells (cohort column + measure columns), keyed by pid. JSON object keys are + # strings; the client indexes with String(pid). + workspace["derived"] = {str(pid): cells for pid, cells in g["derived"].items()} + # Who is looking (permissions verdicts) + the assignee choices for `user`-typed columns — + # the two other host-only render args the shell was missing (fail-closed without them: + # restricted fields uneditable, assignee picker empty). + workspace["viewer"] = {"name": session.uname, "isAdmin": bool(session.admin)} + try: + from core import users as _users + workspace["userOptions"] = _users.assignable_people(tenant=session.tenant) + # Wave 14 C-AVATAR — the options vocabulary's companion: display name -> data URL. + # Absent entries fall back to the client's initials disc; a non-data: value is + # dropped (defence in depth beside the write-side wall in routes_auth). + workspace["userAvatars"] = {k: v for k, v in + _users.avatar_map(tenant=session.tenant).items() + if str(v).startswith("data:image/")} + except Exception: + workspace["userOptions"] = [] + workspace["userAvatars"] = {} + + # THE SURFACE STAMP — a MIRROR of the host's own (`_table_grid`'s hide/cohort stamps). + # ⚠ Deliberately NOT `hideViews`: `cohortMode` is what swaps the Views panel for the cohort + # list, and two switches for one behaviour would drift. + workspace["scopeKey"] = scope + if scope == "cohort": + workspace["cohortMode"] = True + # The create pane offers "this cohort table only" (default) vs "all customer tables". + workspace["scopeChoice"] = True + return {"workspace": workspace} + + +# ─────────────────────────────────────────────── the TIME-SERIES channel (C-TS, 2026-08-02) +_TS_BUCKETS = ("week", "month", "quarter", "year") +_TS_MAX_BUCKETS = 120 +_TS_MAX_FIELDS = 12 +_TS_MAX_PIDS = 5000 +_TS_MAX_LAST_N = 120 +#: C-TSWIN (wave 14): sliding windows cost ONE aggregate query per (metric, bucket) — the +#: price of "bucket N == the grid cell at bucket N's end". The product cap keeps a legal +#: 120-bucket × 12-metric ask from becoming 1,440 queries in one request; typical panels +#: (12 × 3) sit two orders of magnitude under it. Dated amendment in the split doc. +_TS_MAX_CELLS = 720 + + +def _ts_start_of(bucket, d): + """The calendar START of the bucket holding `d` (week = Monday, the vocabulary rule).""" + if bucket == "week": + return d - dt.timedelta(days=d.weekday()) + if bucket == "month": + return d.replace(day=1) + if bucket == "quarter": + return d.replace(month=((d.month - 1) // 3) * 3 + 1, day=1) + return d.replace(month=1, day=1) + + +def _ts_next(bucket, d): + if bucket == "week": + return d + dt.timedelta(days=7) + if bucket == "year": + return d.replace(year=d.year + 1) + step = 1 if bucket == "month" else 3 + m = d.month + step + return dt.date(d.year + (m - 1) // 12, (m - 1) % 12 + 1, 1) + + +def _ts_prev(bucket, d): + if bucket == "week": + return d - dt.timedelta(days=7) + if bucket == "year": + return d.replace(year=d.year - 1) + step = 1 if bucket == "month" else 3 + y, m = d.year, d.month - step + while m < 1: + m += 12 + y -= 1 + return dt.date(y, m, 1) + + +def _ts_label(bucket, start): + if bucket == "week": + return f"Wk of {start.strftime('%b %d')}" + if bucket == "month": + return start.strftime("%b %Y") + if bucket == "quarter": + return f"Q{(start.month - 1) // 3 + 1} {start.year}" + return str(start.year) + + +@router.post("/grid/timeseries") +def grid_timeseries(body: dict = Body(default=None), scope: str = "customer", + session: Session = Depends(module_gate(MODULE))): + """Pooled measure values per calendar bucket — the time-series view's data channel. + + C-TSWIN (wave 14, ruling R1): AS-OF semantics. Each metric's OWN stored window is + re-resolved per bucket with `today := min(bucket end, real today)` — `ytd` is cumulative + from Jan 1, `last_90_days` trailing, `all_time` cumulative ever; bucket N equals what the + grid's measure column would show if today were bucket N's end. Fixed-range (`custom`) + windows cannot slide and are dropped per field as `window_fixed`. Rows carry + `window: {kind, label}` so a cumulative row cannot be misread as periodic, and there is + NO `total` column — sliding windows overlap, so a sum of columns would double-count. + + The client sends the pids its view currently matches (the filter IS the scope); the server + intersects them with the session's book, so the request can only ever NARROW. Values are + POOLED aggregates computed by the semantic layer's own expression (the additivity law: an + average is computed at pool grain, never averaged over per-customer answers). A bucket + with no rows is 0 for a sum/count and null for anything else; a bucket that has not + STARTED yet is null for every kind — an unstarted period's YTD is unanswered, not 0. + """ + from core import measure_resolve + from harness import windows as _wn + from routes_customers import _pool_stamp, _team_agent, allowed_pids + + # Wave 16 C-TOPIC: the channel is CUSTOMER-GRAIN (measure_resolve's own grain), so the + # product surface is refused in words rather than answered with an id-space accident — + # product pids are CRC32 hashes and would mostly fall outside the customer book anyway, + # but "mostly" is not a wall. The client hides the mode for this topic; this is the + # server's half of the same refusal. + _sc = _scope_or_400(scope) + if _sc == "product" or _sc.startswith("ut_"): + raise err(400, "bad_timeseries", + "this surface has no time-series channel — measures are customer-grain") + + body = body or {} + bucket = body.get("bucket") + if bucket not in _TS_BUCKETS: + raise err(400, "bad_timeseries", "bucket must be one of week, month, quarter, year") + raw_fields = body.get("fields") + if not isinstance(raw_fields, list) or not raw_fields: + raise err(400, "bad_timeseries", "fields must be a non-empty list of field keys") + if len(raw_fields) > _TS_MAX_FIELDS: + raise err(400, "bad_timeseries", f"at most {_TS_MAX_FIELDS} fields per request") + raw_pids = body.get("pids") + if not isinstance(raw_pids, list) or not raw_pids: + raise err(400, "bad_timeseries", "pids must be a non-empty list") + if len(raw_pids) > _TS_MAX_PIDS: + raise err(400, "bad_timeseries", f"at most {_TS_MAX_PIDS} pids per request") + try: + wanted = {int(p) for p in raw_pids} + except (TypeError, ValueError): + raise err(400, "bad_timeseries", "pids must be integers") + pool = wanted & {int(p) for p in allowed_pids(session)} + if not pool: + # 403, not an empty 200: the caller asked about customers outside their book, and an + # all-zero series over nobody would read as "no activity", not "not yours". + raise err(403, "out_of_scope", "none of those customers are in your book") + + span = body.get("span") + if not isinstance(span, dict): + raise err(400, "bad_timeseries", "span must be {'lastN': n} or {'from': .., 'to': ..}") + today = time.strftime("%Y-%m-%d") + t = dt.date.fromisoformat(today) + n = span.get("lastN") + starts = [] + if n is not None: + if not isinstance(n, int) or isinstance(n, bool) or not (1 <= n <= _TS_MAX_LAST_N): + raise err(400, "bad_timeseries", f"lastN must be 1..{_TS_MAX_LAST_N}") + cur = _ts_start_of(bucket, t) + starts = [cur] + for _ in range(n - 1): + cur = _ts_prev(bucket, cur) + starts.append(cur) + starts.reverse() + else: + try: + d_from = dt.date.fromisoformat(str(span.get("from"))) + d_to = dt.date.fromisoformat(str(span.get("to"))) + except (TypeError, ValueError): + raise err(400, "bad_timeseries", "span.from/to must be ISO dates (YYYY-MM-DD)") + if d_from > d_to: + d_from, d_to = d_to, d_from + cur = _ts_start_of(bucket, d_from) + while cur <= d_to: + starts.append(cur) + if len(starts) > _TS_MAX_BUCKETS: + raise err(400, "bad_timeseries", + f"that span is more than {_TS_MAX_BUCKETS} {bucket} buckets - " + f"narrow it") + cur = _ts_next(bucket, cur) + if not starts: + raise err(400, "bad_timeseries", "the span holds no buckets") + ends = [_ts_next(bucket, s) - dt.timedelta(days=1) for s in starts] + + rt = session.runtime + if not rt.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable") + import modules.customer_data as cl_mod + # Read-only route: it must not consume a one-shot label-correction ack the browser has + # not seen yet (the same rule event validation follows). + ws = cl_mod.table_workspace(session.uname, consume_corrections=False) + fdefs = ws.get("fields") or {} + keys, dropped, seen = [], [], set() + for k in raw_fields: + k = str(k or "") + if not k or k in seen: + continue + seen.add(k) + fd = fdefs.get(k) + if not isinstance(fd, dict) or not isinstance(fd.get("measure"), dict): + # Named, never silent: v1 eligibility is measure-backed fields only (C-TS). + dropped.append({"field": k, "reason": "not_a_measure_field"}) + continue + keys.append(k) + # ⚡ WAVE 17 R9 (amendment 2026-08-03, GRID's cross-fence ask) — A SHEET WITH NO MEASURE + # FIELDS IS NO LONGER A 400. It answers with the BUCKET GRID and no rows. + # + # Why this is the honest direction and not a loosening: the bucket starts and their labels + # are SERVER math (`_ts_start_of` / `_ts_next` / `_ts_label`), and R9 has the client + # synthesizing snapshot rows for preset columns that have no window to slide. Those rows + # must be painted under the SAME headings as everything else, so the client needs the + # columns even when the server has no series to put in them. Refusing the whole request + # meant the panel could offer nothing at all on this tenant — the shipped contract has zero + # measure-backed presets (`aios_grid.py`: "this branch is currently MEMBERLESS"), which is + # what item 5 is actually about. + # + # Nothing is invented by this: `rows` is empty and `meta.dropped` NAMES every field and why. + # A 400 is still returned for a request that is malformed (bad bucket, bad span, no fields + # at all) — this only stops treating "I asked about columns you cannot serve" as an error. + mfields, slid_keys = [], [] + for k in keys: + m = dict(fdefs[k]["measure"]) + if (m.get("window") or {}).get("kind") == "custom": + # C-TSWIN: a fixed date range cannot slide across buckets — named, never silent, + # the same posture as not_a_measure_field. + dropped.append({"field": k, "reason": "window_fixed"}) + continue + mfields.append({"key": k, "measure": m}) + slid_keys.append(k) + # Same amendment as above: every metric being fixed-range is a sheet with no SERIES, not a + # broken request. The columns still stand, and `window_fixed` still names each refusal. + if len(starts) * len(mfields) > _TS_MAX_CELLS: + raise err(400, "bad_timeseries", + "that ask is too wide - narrow the span or pick fewer metrics") + + team_id, agent = _team_agent(session) + stamp = _pool_stamp(rt, team_id, agent) + problems = [] + bucket_bounds = [(s.isoformat(), e.isoformat()) for s, e in zip(starts, ends)] + # No slidable metrics = nothing to resolve. Skipping the call rather than asking the + # resolver about an empty list keeps the memo free of a meaningless key. + answers = measure_resolve.series_values( + mfields, bucket, bucket_bounds, + team_id, frozenset(pool), today, stamp, rt.series_memo, + on_error=lambda tag, e: problems.append(str(e)[:200])) if mfields else {} + + columns = [] + for s, e in zip(starts, ends): + col = {"key": s.isoformat(), "label": _ts_label(bucket, s), + "from": s.isoformat(), "to": e.isoformat()} + if e > t: + col["partial"] = True + columns.append(col) + rows = [] + for k in slid_keys: + ans = answers.get(k) + if ans is None: + dropped.append({"field": k, "reason": "unresolvable"}) + continue + vals_by = ans.get("values") or {} + agg_kind = str(ans.get("agg") or "sum") + zero_fill = agg_kind in ("sum", "count") + vals = [] + for s in starts: + if s > t: + # C-TSWIN: an unstarted period is unanswered for EVERY agg kind — a future + # month's YTD zero-filled to 0 would read as "the year reset". + vals.append(None) + else: + vals.append(vals_by.get(s.isoformat(), 0 if zero_fill else None)) + wspec = (fdefs[k].get("measure") or {}).get("window") + wnorm = _wn.normalize(wspec) or {} + rows.append({"field": k, "label": str(fdefs[k].get("label") or k)[:120], + "agg": agg_kind, "values": vals, + "window": {"kind": str(wnorm.get("kind") or ""), + "label": _wn.label(wspec)}}) + meta = {"pool": len(pool), "today": today, "bucket": bucket} + if dropped: + meta["dropped"] = dropped + if problems: + meta["problems"] = problems[:5] + return {"columns": columns, "rows": rows, "meta": meta} + + +#: C-CAL caps (wave 17, owner item 9 / ruling R4). A month of days, a handful of metrics. +#: Every one of these is a 400 WITH ITS REASON, never a silent trim: a calendar that quietly +#: answered 20 of the 31 days asked about would paint eleven blank cells that look like days +#: with no activity. +_CAL_MAX_GROUPS = 31 +_CAL_MAX_FIELDS = 6 +_CAL_MAX_CELLS = 186 + + +@router.post("/grid/calendar_metrics") +def grid_calendar_metrics(body: dict = Body(default=None), scope: str = "customer", + session: Session = Depends(module_gate(MODULE))): + """Per-DAY measure values with each metric's own window slid to that day (C-CAL / R4). + + ⛔ WHY THIS EXISTS AT ALL. The calendar's summary cells used to aggregate the row VALUES the + grid already held — which for a measure column means "every member's YTD **as of today**", + summed and printed under a date in March. The number was arithmetically fine and semantically + a lie: it answered a question about today while sitting in a cell labelled with another day. + R4: a metric in a day cell is computed AS OF THAT DAY. + + The shape differs from the time-series channel in exactly one way, and it is the reason this + is a separate route rather than a parameter: **every group carries its OWN pid set**. A + calendar day holds the records the date field placed there, so day-to-day the subject + changes. One `allowed_pids` for the whole request — the TS channel's shape — would compute + each day over everybody, which is a different question again. + + Static (non-measure) fields are NOT served here. They have no window to slide, so the + client's own per-day aggregation over row values stays correct for them; sending them would + invite a second implementation of arithmetic that already works. + """ + from core import measure_resolve + from harness import windows as _wn + from routes_customers import _pool_stamp, _team_agent, allowed_pids + + # The same refusal the TS channel makes, for the same reason: measures are customer-grain + # and product pids are CRC32 hashes of SKU codes. "Mostly outside the book" is not a wall. + _sc_cal = _scope_or_400(scope) + if _sc_cal == "product" or _sc_cal.startswith("ut_"): + raise err(400, "bad_calendar_metrics", + "the product surface has no measure channel yet — measures are customer-grain") + + body = body or {} + raw_groups = body.get("groups") + if not isinstance(raw_groups, list) or not raw_groups: + raise err(400, "bad_calendar_metrics", + "groups must be a non-empty list of {key: 'YYYY-MM-DD', pids: [...]}") + if len(raw_groups) > _CAL_MAX_GROUPS: + raise err(400, "bad_calendar_metrics", + f"at most {_CAL_MAX_GROUPS} days per request (one month)") + raw_fields = body.get("fields") + if not isinstance(raw_fields, list) or not raw_fields: + raise err(400, "bad_calendar_metrics", "fields must be a non-empty list of field keys") + if len(raw_fields) > _CAL_MAX_FIELDS: + raise err(400, "bad_calendar_metrics", f"at most {_CAL_MAX_FIELDS} metrics per request") + if len(raw_groups) * len(raw_fields) > _CAL_MAX_CELLS: + raise err(400, "bad_calendar_metrics", + "that ask is too wide - fewer days or fewer metrics") + + book = {int(p) for p in allowed_pids(session)} + groups, seen_days = [], set() + for g in raw_groups: + if not isinstance(g, dict): + raise err(400, "bad_calendar_metrics", "every group must be an object") + day = str(g.get("key") or "") + try: + d = dt.date.fromisoformat(day) + except (TypeError, ValueError): + raise err(400, "bad_calendar_metrics", + "every group key must be an ISO date (YYYY-MM-DD)") + if day in seen_days: + raise err(400, "bad_calendar_metrics", f"day {day} appears twice") + seen_days.add(day) + raw_pids = g.get("pids") + if not isinstance(raw_pids, list): + raise err(400, "bad_calendar_metrics", "every group needs a pids list") + try: + wanted = {int(p) for p in raw_pids} + except (TypeError, ValueError): + raise err(400, "bad_calendar_metrics", "pids must be integers") + # NARROW-ONLY, per group. The client sends what its calendar placed; the server can + # only ever remove from that, never add. + groups.append((day, d, frozenset(wanted & book))) + if sum(len(p) for _, _, p in groups) > _TS_MAX_PIDS: + raise err(400, "bad_calendar_metrics", + f"at most {_TS_MAX_PIDS} customer references per request") + + rt = session.runtime + if not rt.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable") + import modules.customer_data as cl_mod + ws = cl_mod.table_workspace(session.uname, consume_corrections=False) + fdefs = ws.get("fields") or {} + keys, dropped, seen = [], [], set() + for k in raw_fields: + k = str(k or "") + if not k or k in seen: + continue + seen.add(k) + fd = fdefs.get(k) + if not isinstance(fd, dict) or not isinstance(fd.get("measure"), dict): + # The TS channel's vocabulary, deliberately reused rather than re-coined: the client + # already knows how to say these three words to a reader. + dropped.append({"field": k, "reason": "not_a_measure_field"}) + continue + if ((fd["measure"].get("window") or {}).get("kind") == "custom"): + dropped.append({"field": k, "reason": "window_fixed"}) + continue + keys.append(k) + if not keys: + raise err(400, "bad_calendar_metrics", + "none of the requested fields are measure fields with a window that can " + "slide to a day") + + today = time.strftime("%Y-%m-%d") + t = dt.date.fromisoformat(today) + team_id, agent = _team_agent(session) + stamp = _pool_stamp(rt, team_id, agent) + problems = [] + values = {k: {} for k in keys} + for day, d, pids in groups: + # A day that has not happened is unanswered for EVERY aggregate kind — the unstarted + # bucket law at day grain. Answering 0 would say "we sold nothing", which is a claim + # about a day nobody has lived through yet. + if d > t or not pids: + for k in keys: + values[k][day] = None + continue + answers = measure_resolve.series_values( + [{"key": k, "measure": dict(fdefs[k]["measure"])} for k in keys], + "day", [(day, day)], team_id, pids, today, stamp, rt.series_memo, + on_error=lambda tag, e: problems.append(str(e)[:200])) + for k in keys: + ans = answers.get(k) + if ans is None: + values[k][day] = None + continue + vals_by = ans.get("values") or {} + zero_fill = str(ans.get("agg") or "sum") in ("sum", "count") + values[k][day] = vals_by.get(day, 0 if zero_fill else None) + + for k in keys: + if all(v is None for v in values[k].values()): + # Every day unanswerable is a FIELD-level failure, and saying so is the difference + # between "no activity that month" and "this metric could not be computed". + dropped.append({"field": k, "reason": "unresolvable"}) + out = {"values": values, "today": today, + "windows": {k: {"kind": str((_wn.normalize( + (fdefs[k].get("measure") or {}).get("window")) or {}).get("kind") or ""), + "label": _wn.label((fdefs[k].get("measure") or {}).get("window"))} + for k in keys}} + if dropped: + out["dropped"] = dropped + if problems: + out["problems"] = problems[:5] + return out + + +#: ⭐⭐ THE BULK CELL DOOR'S OWN CEILING, and it is REPORTED rather than silently enforced. The +#: standing no-cap rule governs data read from a connected source; this is a WRITE of team-typed +#: values, so a bound is right — but R6's second sentence still binds, which is why exceeding it +#: is a 400 naming the number and the count, never a truncation. +#: Sized against the real job with headroom: the 2027 catalog is 1,397 rows. +MAX_BULK_ROWS = 10_000 + + +@router.post("/grid/bulk-cells") +def bulk_cells(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """`{scopeKey, rows: {"": {key: value}}}` → shared cells on many rows, ONE transaction. + + ⭐⭐ WHY THIS EXISTS AT ALL, because "we already have `/grid/events`" is the obvious objection. + That door is capped at `_MAX_EVENTS = 24` (it is sized for the client's resend window), so a + 1,397-row import is ~59 sequential POSTs against ONE JSON document — and this repo has a + MEASURED scar for exactly that: 18 writes against one document under the store's coalescing + single-flight landed **zero** while answering 200 eighteen times. Raising `_MAX_EVENTS` would + have widened the browser's own resend window to fix a script's problem. One door, one + transaction, is the honest shape. + + ⛔ TENANT-WIDE KEYS ONLY, AND THAT IS THE POINT RATHER THAN A LIMITATION. It writes through + `shared_overlay`, which every account in the tenant reads. A per-user bulk write would be a + contradiction: nobody imports 1,397 rows of the team's work so that one account can see it — + that is precisely the failure owner ruling R6 was made to avoid. A key that is not declared + `shared: true` in the canonical contract is REFUSED BY NAME. + + ⛔ ADMIN ONLY. This mutates data every account in the tenant reads, in bulk, in one call. + D-172 already records that the shared stratum has an asymmetric wall (anyone may create a + tenant-wide column, only the creator or an admin may delete one); a new door does not get to + inherit the loose half of an asymmetry somebody already flagged. + """ + import core.shared_overlay as shared_overlay + import modules.product_data as pd + import core.perm_scope as perm_scope + from routes_products import MODULE as PRODUCT_MODULE, scoped_pool + + scope = _scope_or_400((body or {}).get("scopeKey")) + if scope != "product": + # Fail-closed with the reason: the shared stratum is a PRODUCT-topic mechanism today. + raise err(400, "scope_not_bulk_writable", + f"bulk cell writes are available on the product database; '{scope}' has no " + f"tenant-wide cell stratum") + session.require(PRODUCT_MODULE) + if not session.admin: + raise err(403, "admin_only", + "a bulk write changes values every account in this workspace reads") + + rows = (body or {}).get("rows") + if not isinstance(rows, dict): + raise err(400, "bad_rows", "expected {rows: {\"\": {field: value}}}") + if len(rows) > MAX_BULK_ROWS: + raise err(400, "too_many_rows", + f"at most {MAX_BULK_ROWS} rows per request; this one carried {len(rows)}") + + pids, _team, _src, fields_base = scoped_pool(session) + allowed = {int(p) for p in pids} + shared_keys = set(pd.SHARED_KEYS()) + hidden = perm_scope.hidden_keys(session.user, PRODUCT_MODULE, fields_base) + + clean, unknown_pid, refused_keys = {}, [], {} + for raw_pid, values in rows.items(): + try: + pid = int(raw_pid) + except (TypeError, ValueError): + unknown_pid.append(str(raw_pid)[:40]) + continue + # ⚠ THE POOL IS THE WALL. `scoped_pool` is the same predicate the read door uses, so a + # caller cannot write a row they could not see — including a row in another BU. + if pid not in allowed: + unknown_pid.append(str(raw_pid)[:40]) + continue + keep = {} + for key, value in dict(values or {}).items(): + key = str(key) + if key not in shared_keys: + refused_keys.setdefault("not_shared", set()).add(key) + continue + if key in hidden: + refused_keys.setdefault("hidden_by_permissions", set()).add(key) + continue + if not isinstance(value, (str, int, float)) or isinstance(value, bool): + refused_keys.setdefault("unsupported_value", set()).add(key) + continue + keep[key] = value + if keep: + clean[pid] = keep + + written = shared_overlay.put_rows(pd.TABLE_KEY, clean, st=pd.TABLE_OPS.st) if clean else {} + out = { + "rows_written": len(written), + "cells_written": sum(len(v) for v in written.values()), + "rows_requested": len(rows), + } + # ⭐ R6's second sentence: what did NOT land, and why. A bulk door that reports only its + # successes is how 701 unmatched SKUs disappear quietly. + if unknown_pid: + out["rows_not_in_your_pool"] = {"count": len(unknown_pid), + "sample": sorted(unknown_pid)[:10]} + if refused_keys: + out["refused_fields"] = {reason: sorted(keys) + for reason, keys in refused_keys.items()} + return out + + +@router.post("/grid/events") +def grid_events_route(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """`{events: []}` → `{results, doc?, toast?}`. + + Wave 21 C4: session-only dependency, per-scope gate below — the write door must admit the + same sessions the read door (`/workspace`) admits, or a tenant without the customer module + can SEE its own user tables and not write to them.""" + from core import grid_events + from routes_customers import grid_assembly + + events = (body or {}).get("events") + if events is None and isinstance(body, dict) and body.get("type"): + events = [body] # a single event object, the legacy shape + if not isinstance(events, list): + raise err(400, "bad_events", "expected {events: [...]}") + if len(events) > _MAX_EVENTS: + raise err(400, "too_many_events", + f"at most {_MAX_EVENTS} events per request (the client's resend window)") + + # Validated with the SAME predicate as the read route. An unrecognised scopeKey used to be + # passed through verbatim, and `core.grid_events` only ever compares it to 'cohort' — so a + # typo degraded silently to customer-scope behaviour on a WRITE. Read and write must agree on + # what a scope is, or the surface you read is not the surface you wrote. + scope = _scope_or_400((body or {}).get("scopeKey")) + # Wave 21 C4 — same per-scope gate as /workspace (read and write doors must agree). + if not (scope == "product" or scope.startswith("ut_")): + session.require(MODULE) + # This assembly validates the write; it is not a payload the browser will render. Leave + # one-shot field-name correction acks queued for the subsequent /workspace refresh. + # Wave 16 C-TOPIC: the PRODUCT topic gets the product assembly — product field contract, + # product pids, and (below) the product TABLE OPS, so a product event is validated against + # and lands in the product bucket. The measure/cohort context is honestly EMPTY there: + # `clean_measure_field` refuses measure creates on this surface by construction (the + # customer-grain descope), which is the fail-closed shape, not an accident. + if scope == "product": + from routes_products import MODULE as PRODUCT_MODULE, product_assembly + + session.require(PRODUCT_MODULE) # the product wall (dependency is session-only, C4) + g = product_assembly(session, consume_corrections=False) + elif scope.startswith("ut_"): + # Wave 18 C3-UT — the user-table wall (creator/admin) is inside the assembly; the + # measure/cohort context is honestly EMPTY (customer-grain machinery, no meaning here). + # ⭐⭐ WAVE 30 / W30-T30 (owner item 2: *"when I click hide fields it crash… no matter the + # size"*). This used to be `ut_assembly(...)`, which builds the whole table to validate a + # write it then throws away: `scoped_pool` allocates a dict per row and sorts them, so ONE + # hide-fields checkbox rebuilt ~33k order rows before the event was even dispatched. + # ⛔ NOTHING IS VALIDATED LESS. The comment above still holds — this assembly is the + # permission wall and the admission context — and `ut_write_ctx` returns the SAME six keys + # this route reads, with a pid set derived from exactly the row ids `scoped_pool` would + # have kept. What it does not do is materialise the rows nobody here looks at. + from routes_tables import ut_write_ctx + + g = ut_write_ctx(session, scope) + # ⭐⭐ W31-T20 / D-174 — A WRITE THAT NEEDS THE ROW SET REFUSES OUT LOUD, and this is the + # half that is easy to skip because the code already "fails closed" without it. When a + # read-through grid's population exceeds one window the pid set is EMPTY, and every + # pid-bearing handler in `grid_events` then answers `False` — `overlay_patch` and + # `add_to_list` both `return False` for a pid not in `allowed_pids`. On the wire that is + # `rerender: false` and HTTP 200: a write the user watched succeed, that did nothing + # ([[lost-write-looks-like-failed-read]]). Schema-only events — hide a field, save a view, + # rename a column — name no pid and are untouched, which is D-170. + # ⚠ THE TEST IS THE EVENT'S OWN KEYS, not a kind list: a new pid-bearing event type would + # otherwise inherit the silent no-op the day it is added. + if g.get("limits"): + named = [e for e in events + if isinstance(e, dict) and (e.get("pid") is not None or e.get("pids"))] + if named: + lim = g["limits"][0] + raise err(409, "pid_scope_unresolved", + f"this database is served through the connector mirror and its rows " + f"cannot be listed in one window, so a change addressed to particular " + f"records ({len(named)} of {len(events)} here) cannot be admitted — " + f"{lim.get('cause') or 'the row set is unresolved'}. " + f"{lim.get('recommendation') or ''}".strip()) + else: + g = grid_assembly(session, scope=scope, consume_corrections=False) + # ⚠ THE MEASURE CONTEXT IS NOT OPTIONAL (2026-07-31). Without `measure_offer`, + # `clean_measure_field` had an empty admission list and every measure-column create over + # HTTP was silently refused; without `measure_keys`, `clean_filter_tree` stripped every + # measure CONDITION out of a saved view. The embed always passed these; the API adapter + # simply had not been given them — the standalone shell could read measures it could + # never write. + ctx = _ctx(session, g["fields"], g["pids"], scope_key=scope, + measure_keys=frozenset(m["key"] for m in g["measures"]), + resolved_ids=frozenset(g["measure_sets"]), + cohort_ids=frozenset(c["id"] for c in g["lists"]), + measure_offer=tuple(g["measures"]), + visible_views=tuple(g["views"])) + + # Per-event results so the client can tell which of a batch landed — the component's own + # bridge has no response channel at all, so this is strictly more than the embed gets. + results = [] + try: + for one in events: + eid = str(one.get("id") or "") if isinstance(one, dict) else "" + # ⭐⭐ D-291 — WHICH event was refused, not merely THAT something was. The handler + # appends to a shared list, so the refusals belonging to THIS event are exactly the + # ones that appeared across THIS call. Reading the list once after the loop would + # answer "the batch was refused" and leave the caller to guess which member, which is + # the same class of unfalsifiable answer the channel exists to end. + _before = len(ctx.out.refusals) + rerender = grid_events.handle_one(one, ctx) + row = {"id": eid, "rerender": bool(rerender)} + mine = ctx.out.refusals[_before:] + if mine: + # `refused` is the SHAPE a caller branches on; the first reason is the one that + # stopped this write (a handler returns at its first refusal). + row["refused"] = mine[0] + results.append(row) + except grid_events.StoreUnavailable: + raise err(503, "store_unavailable", + "the tenant store is unavailable — none of your changes were saved") + + out = {"results": results, "rerender": any(r["rerender"] for r in results)} + if ctx.out.refusals: + # ⚠ ALSO AT THE TOP LEVEL, because a batch that was wholly refused must not read as a + # batch that wholly landed. A caller that only checks the envelope still learns something + # is wrong, and a caller that walks `results` learns exactly which member. + out["refusals"] = list(ctx.out.refusals) + if ctx.out.doc is not None: + # ⭐ C4 / W30-T27 — `docPayload` IS THE NAME THE CLIENT ALREADY DECLARES. `types.ts` has + # carried `docPayload?: {pid, docId, name, mime, data_b64}` since C5, and `Documents.tsx` + # matches it against the fetch it is waiting on — while this route has been answering + # `doc`, which `apiBridge.ts` deliberately drops. One object, emitted under the name the + # consumer looks for, so F's wiring needs no translation step to get wrong. + # ⚠ `doc` stays for one wave: nothing in the client reads it, but a gate might, and + # removing a key to save six bytes is not worth a red nobody predicted. + out["doc"] = out["docPayload"] = ctx.out.doc + if ctx.out.toast is not None: + out["toast"] = ctx.out.toast + + # ── ⭐ owner item 2 (2026-08-03): THE NEW MEASURE COLUMN'S VALUES, ONE ROUND TRIP SOONER ── + # + # Creating a measure column cost the browser TWO sequential trips before a single number + # appeared: this one to persist the field, then a whole `/workspace` to compute it. The + # second cannot start until the first lands (the resolver reads the PERSISTED field), so the + # wait was structural, not slow code — the owner's "it takes some time for the data to + # populate". The values are computed here instead, immediately after the write, and ride + # this response. + # + # ⚠ IT COSTS NOTHING EXTRA TO COMPUTE. The expensive part is one DuckDB aggregate over the + # book, and `rt.measure_memo` is keyed on (pool stamp, scope, pool, measure, window) — so + # the `/workspace` re-read that still follows HITS the memo instead of doing this work. The + # query happens once either way; only its position moved. + # + # ⚠ NARROW ON PURPOSE. Gated to an actual measure-column write, so an overlay edit or a + # cohort add — the overwhelming majority of events — never pays for a second assembly. + # + # ⚠ AND IT IS A SHORTCUT, NOT A PATH. Any failure is swallowed: `WORKSPACE_STALE` still + # fires from `rerender`, and the re-read still delivers these values exactly as it does + # today. Nothing depends on this having worked. + if out["rerender"] and scope in ("customer", "cohort") and any( + isinstance(e, dict) and e.get("type") == "field_upsert" + and str(((e.get("field") or {}) if isinstance(e.get("field"), dict) else {}) + .get("key") or "").startswith("measure_") + for e in events): + try: + fresh = grid_assembly(session, scope=scope, consume_corrections=False) + out["derived"] = {str(pid): cells for pid, cells in fresh["derived"].items()} + except Exception: + pass + # ⚠ NOTHING IS INVALIDATED HERE, on purpose. The runtime cache holds ONLY the scope-shaped + # Odoo pool (see `routes_customers._pool_rows`), and no event on this route can change an + # Odoo column — Odoo is read-only. Everything an event DOES change (overlays, fields, views, + # folders, cohorts) is re-read from the store on the next request. An earlier version cleared + # `pool_cache` after an `overlay_patch`, which threw away an expensive Odoo pull to refresh + # data that was never in it. + return out + + +# ── CONTRACT C1 (W36-T20): THE REGISTRY TOPICS' ROW READERS ─────────────────────────────────── +# ⭐⭐ R6 — *"EVERY database gets the same permission logic, always."* `core.perm_scope.scoped_table` +# is the ONE door to any database's rows, and it cannot import a topic's pool builder: `core` never +# imports up (`platform/ARCHITECTURE.md`) and these pools are built by `modules/` + `aios_grid` +# behind this layer's per-tenant cache. So the app layer DECLARES its readers, exactly as +# `routes_odoo_tables` declares connected tables to `user_tables.register_connected`. +# +# ⛔ REGISTERED HERE RATHER THAN IN `routes_customers`/`routes_products` because those two files +# are outside wave 36's lane-C fence. The readers themselves are three lines each and call the +# SAME `_pool_for` + `derive_pool_scope` pair those routes call, so there is no second pool and no +# second scope derivation — only a second CALLER of the one that exists. +# +# ⚠ AND THE TOPIC ROUTES STILL HAVE THEIR OWN DOOR TODAY. Contract C1 says `apply_row_scope` + +# `visible_fields` "move behind" `scoped_table`; moving `routes_customers.grid_assembly` and +# `routes_products.scoped_pool` is booked as a PENDING row (mailbox/C.md, C-1) rather than done +# here, because neither file is in this fence. What ships now is the arm the wave is load-bearing +# on — every `ut_*` database, plus E's sandbox — and a topic arm that is REAL rather than stubbed, +# so `scoped_table`'s topic leg is exercised by the product instead of only by a gate. +def _topic_rt(st, module): + """The tenant runtime a topic pool must be built against, or a REPORTED refusal. + + ⛔ A topic pool is per TENANT (`rt.pool_cache`), so `st=None` cannot be resolved to "the + default" without picking a tenant at random — which on this box is tenant #0's PRODUCTION + data. Standing rule 1's second sentence: say why, and say what to do instead. + """ + if st is None: + import core.perm_scope as perm_scope + raise perm_scope.Unresolvable( + subject="rows", effect="unreadable", + cause=f"'{module}' is a registry topic whose pool is built per tenant and no tenant " + f"runtime was passed", + recommendation="pass the session's runtime as `st=`. A topic pool cannot be " + "resolved without knowing which tenant is asking") + return st + + +def _customer_rows(table_key, user, st): + """`(fields, rows)` for the customer topic — the SAME derivation `_team_agent` uses.""" + import aios_grid + import core.perm_scope as perm_scope + from routes_customers import _pool_for + + rt = _topic_rt(st, table_key) + team_id, agent = perm_scope.derive_pool_scope(user, table_key) + return list(aios_grid.FIELDS), _pool_for(rt, team_id, agent) + + +def _product_rows(table_key, user, st): + """`(fields, rows)` for the product topic. `consolidated=` follows the derived scope, so a + BU-pinned reader gets that BU's field contract rather than the consolidated one.""" + import core.perm_scope as perm_scope + from routes_products import _pool_for, pd_fields + + rt = _topic_rt(st, table_key) + team_id, _agent = perm_scope.derive_pool_scope(user, table_key) + return pd_fields(consolidated=team_id is None), _pool_for(rt, team_id) + + +def _register_topic_rows(): + """Declare both topic readers to C1. Called at import; returns the registered key set. + + ⚠ THE KEYS ARE LITERALS AND THE ROUTE IMPORTS ARE INSIDE THE READERS, on purpose: this runs at + module import, and `from routes_products import MODULE` here would pull a sibling router in + before its own imports have settled. Every other cross-router reference in this file is lazy + for the same reason. The literals are held to their sources by `verify_scopes`, so they cannot + drift into naming a topic that does not exist. + """ + import core.perm_scope as perm_scope + + perm_scope.register_rows(_customer_rows, MODULE) + return perm_scope.register_rows(_product_rows, _PRODUCT_MODULE) + + +#: ⚠ `_`-prefixed, because three functions in this file already bind the name `PRODUCT_MODULE` +#: LOCALLY from `routes_products`. A module-level twin of that spelling would read as the same +#: thing and be a different one — [[constant-two-features-share]] waiting to happen. +_PRODUCT_MODULE = "product_data" + +_C1_ROW_SOURCES = _register_topic_rows() diff --git a/api/routes_keychain.py b/api/routes_keychain.py index 0d8534c02e4acf8e5318379694ff5a39602887f5..4ef61ca06030bf7abf2238ae1421cbde6686560d 100644 --- a/api/routes_keychain.py +++ b/api/routes_keychain.py @@ -1,1073 +1,1073 @@ -"""routes_keychain.py — Keychains + Connectors admin surfaces (wave 18, C7 / R3). - -Keychain: encrypted per-tenant credential entries (`core/keychain.py`). The routes NEVER -return a decrypted field — list rows carry a masked preview, and the decrypt function is a -connector-layer internal. Connectors: the tenant's data sources as STATUS rows — Royal's -env-configured Odoo, keychain-held sources — plus R3's guardrail: the **Unsynced records** -count (rows holding overlay data whose pids the current pool no longer serves; counted and -drillable, never silently dropped) and a pause toggle whose v1 semantics are stated honestly -in the payload (`pausedNote`): pausing marks intent and warns; the source cutover ships with -the keychain cutover wave R3 staged. -""" -import os - -from fastapi import Body, Depends -from fastapi import APIRouter - -from deps import Session, err, require_session -# ⚠ W32-T11 / R4: `admin_gate` is GONE from this module, and its absence is the ruling. Every door -# here was admin-only, which made "a member may hold a personal connection" unbuildable — the wall -# moved from the ROUTE into the ROW (`may_see` / `_may_touch`), where a scope can be enforced per -# entry instead of per endpoint. `session.admin` is still what business-wide requires. - -router = APIRouter(prefix="/api/v1") - -#: ⭐ D-10 (wave 24): ONE literal, owned by the harness. It was spelled here AND implied by -#: `harness/runtime.py`'s reader; a pause flag written under one spelling and read under another -#: freezes nothing while reporting success, which is the shape of the bug D-10 books. -from harness.runtime import (CONNECTOR_FLAGS_KEY as _CONNECTOR_FLAGS_KEY, # noqa: E402 - ENV_ODOO_FLAG_KEY as _ENV_ODOO_FLAG_KEY) -#: DEBT-2 (2026-08-04): the last-successful-sync snapshot bucket. Written when the RESOLVED -#: odoo connector is paused; read by the pool path while paused; survives a Space restart. -SNAPSHOT_KEY = "connector_snapshots" - -#: ⛔⛔ W32-T10 / OWNER ITEM 6 — THE TENANT THAT OWNS THE PROCESS ENVIRONMENT. -#: -#: `ODOO_URL` and its siblings are tenant #0's `.env` / Space secrets, and ONE Space process serves -#: EVERY tenant. So "the environment has Odoo credentials" is a fact about this DEPLOYMENT, and -#: turning it into "your workspace is connected to Odoo" is only true for one slug. That is R3's -#: rule, stated in `harness/runtime.py::odoo_source` as *"NEVER the environment: env is tenant #0's -#: connection, and handing it to another tenant is the leak this method exists to prevent."* -#: -#: ⚠ IT IS SPELLED HERE BECAUSE THE RUNTIME EXPORTS NO CONSTANT FOR IT — `odoo_source` and -#: `odoo_flag_key` both carry the literal. `verify_meta`'s W32-T10 section asserts this value -#: against `runtime.py`'s own text, so the day the runtime's answer changes and this one does not, -#: the gate reds instead of the product quietly disagreeing with itself. -ENV_ODOO_TENANT = "royal-imports" - - -def env_odoo_available(rt): - """Does the PROCESS ENVIRONMENT offer an Odoo connection to THIS tenant? (W32-T10.) - - ⛔ THE ONE NORMALIZER FOR ONE QUESTION, and it exists because there were two answers to it. - `routes_connectors.directory._odoo` read a bare `os.environ.get("ODOO_URL")` with **no tenant - guard** — one process, every tenant — so a nurilab admin opening Connectors was told Odoo was - `connected` and offered "Manage keys" for a credential belonging to another company. This - module asked the same question correctly two functions below, which is the whole shape of - [[one-question-two-normalizers]]: the correct copy hides the wrong one until somebody signs in - as the second tenant. - - ⚠ NOT the same question as `rt.odoo_flag_key() == ENV_ODOO_FLAG_KEY`. That one answers *which - source WINS*, so it goes False the moment a keychain entry exists — correct for a pause flag, - wrong for "should the environment row be listed at all", which is what the connectors pane - needs in order to show an inactive env source beside an active keychain one. - - Never raises: a runtime that cannot answer is not connected. Fail closed — a missing guard is - how another tenant's environment got reported as this tenant's connection in the first place. - """ - try: - if not (os.environ.get("ODOO_URL") or "").strip(): - return False - return str(getattr(rt, "key", "") or "") == ENV_ODOO_TENANT - except Exception: # noqa: BLE001 - return False - - -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# ⭐⭐ W32-T11 / CONTRACT C1 / OWNER RULING R4 — BUSINESS-WIDE vs PERSONAL, ON EVERY CONNECTION -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# -# R4, verbatim: *"The business-wide vs personal split lands on ALL connections, not just Odoo. -# Every keychain entry and connector carries a scope, selectable per connection; business-wide is -# admin-only and applies to every user in the tenant."* -# -# ⛔ THE VOCABULARY IS DECLARED HERE AND NOWHERE ELSE (contract C1, and it names this file -# explicitly). `core/keychain.py` is the ENCRYPTED STORE and belongs to the integrator; a scope is -# an access rule, not a secret, so it lives in the layer that already decides who may ask. -# `verify_meta` asserts this tuple against `connectors/ConnectorsPage.tsx`, so the two halves -# cannot drift into two vocabularies. -SCOPES = ("business", "personal") -#: ⚠ THE READ DEFAULT, AND IT IS A READ DEFAULT — never a write (C1). Every entry stored before -#: this wave was, by construction, an admin's tenant-wide credential, so `business` is not a guess. -#: Materialising it would be a migration nobody authorised and would rewrite the store on the next -#: read [[a-migration-that-runs-on-the-next-write]]. -DEFAULT_SCOPE = "business" -#: The side bucket: `{entry_id: {"scope": "personal", "owner": ""}}`. -#: ⛔ A BUSINESS ENTRY WRITES NO ROW — it IS the default, so an absent row and a `business` row mean -#: the same thing and there is only one way to spell the common case. -SCOPE_KEY = "keychain_scopes" - -#: ⛔⛔ THE TYPES A WHOLE WORKSPACE READS THROUGH, WHICH THEREFORE CANNOT BE PERSONAL. -#: -#: `keychain.odoo_creds` and `keychain.meta_creds` both resolve to *the first entry of that type* -#: for the TENANT — that is what spawns `ut_odoo_*` / `ut_meta_*` and what every measure column is -#: answered from. So a member storing a personal Odoo key would not get "their own Odoo": they -#: would silently become the credential the entire workspace's databases are built from, which is -#: a credential elevation wearing a scope picker. -#: ⚠ REFUSED WITH A REASON, NEVER SILENTLY COERCED TO `business` — a picker that quietly changes -#: your answer is worse than one that says no (W30/R6's second sentence). The resolver itself is -#: `harness/runtime.py` / `core/keychain.py`, the integrator's files; this refusal closes the door -#: from the only side B owns, and the resolver-side guard is booked for A. -TENANT_WIDE_TYPES = ("odoo", "meta_ads") - - -def clean_scope(raw, default=DEFAULT_SCOPE): - """A scope word from the wire, or None when the caller said something we do not speak. - - Distinguishing "said nothing" (⇒ the default) from "said nonsense" (⇒ 400) is the whole - reason this returns None rather than falling back: a typo'd `"personel"` silently becoming - business-wide is exactly the failure a scope picker exists to prevent. - """ - if raw is None or (isinstance(raw, str) and not raw.strip()): - return default - got = str(raw).strip().lower() - return got if got in SCOPES else None - - -def _scope_rows(rt): - try: - return dict(rt.get(SCOPE_KEY) or {}) - except Exception: # noqa: BLE001 - return {} - - -def entry_scope(rt, entry_id, rows=None): - """`(scope, owner)` for one entry. `rows` is the bucket, passed in when walking a list so a - census does not re-read the store once per entry.""" - r = (rows if rows is not None else _scope_rows(rt)).get(str(entry_id)) - if not isinstance(r, dict): - return DEFAULT_SCOPE, "" - return (str(r.get("scope") or DEFAULT_SCOPE), str(r.get("owner") or "")) - - -def may_see(scope, owner, uname): - """R4's visibility rule: business-wide is everyone's, personal is its owner's. - - ⛔ AND AN ADMIN IS NOT AN EXCEPTION. R4 says a personal entry belongs to a person; the - per-user OAuth slots one module down have made the same call since wave 22 (*"a refresh token - is identity, not infrastructure"*). An admin who could read every member's personal credential - would make "personal" a label rather than a boundary. - """ - return scope != "personal" or str(owner) == str(uname) - - -def visible_entries(rt, uname, is_admin=False): - """This USER's view of the keychain: every business entry plus their own personal ones, each - row carrying its `scope` and `owner` so no caller has to ask a second time. - - ⛔ THE MASKED PREVIEW IS NOT PART OF "VISIBLE". R4 opens this room to members so they can - hold a connection of their own; it does not hand them four characters of the workspace's Odoo - key. So a business row a member did not create arrives WITHOUT `preview` — they can see that - the connection exists and is theirs to use, which is the whole of what R4 grants. The default - is the RESTRICTED one deliberately: a caller that forgets the argument leaks nothing. - """ - rows = _scope_rows(rt) - out = [] - for e in _kc().list_entries(rt): - scope, owner = entry_scope(rt, e["id"], rows) - if not owner: - owner = str(e.get("createdBy") or "") - if not may_see(scope, owner, uname): - continue - row = {**e, "scope": scope, "owner": owner} - if not (is_admin or str(owner) == str(uname)): - row["preview"] = "" - out.append(row) - return out - - -def _write_scope(rt, entry_id, scope, owner): - """Persist one entry's scope. A `business` entry CLEARS its row rather than writing the - default, so the store holds one spelling of the common case.""" - def _up(cur): - if scope == DEFAULT_SCOPE: - cur.pop(str(entry_id), None) - else: - cur[str(entry_id)] = {"scope": scope, "owner": str(owner or "")} - return cur - - rt.update(SCOPE_KEY, _up, flush="sync") - return True - - -def _may_touch(session, row): - """May this session change or delete `row`? An admin owns the business-wide ones; a member - owns their own personal ones. Anything else is not theirs to move.""" - if row.get("scope") == "personal": - return str(row.get("owner") or "") == str(session.uname) - return bool(session.admin) - - -def _kc(): - import core.keychain as keychain - return keychain - - -def _resolved_odoo_key(rt): - """`(source, flag_key)` — which source would serve this tenant's Odoo queries, in both the - shapes this module needs: the display string (`env` / `keychain:`) and the key the pause - flag is stored under. - - ⭐ D-10 (wave 24): THE RESOLUTION ITSELF NOW LIVES IN ONE PLACE, `TenantRuntime.odoo_flag_key`, - beside the `odoo_source()` it must agree with. This function had its own copy of the same - three rules — first unlocked keychain odoo entry, else env for tenant #0, else nothing — and - a second copy is exactly how a pause flag comes to be written against one resolution and read - against another, freezing nothing while the UI reports success. The two SHAPES stay here - because they are this module's presentation concern; the DECISION does not. - """ - flag_key = rt.odoo_flag_key() - if not flag_key: - return None, None - return ("env" if flag_key == _ENV_ODOO_FLAG_KEY else f"keychain:{flag_key}"), flag_key - - -def odoo_paused(rt): - """True when the tenant's RESOLVED Odoo source carries the pause flag. Pausing an entry - that is not the resolved source freezes nothing — it serves nothing. - - ⭐ D-10: a thin delegate now. The implementation moved to `TenantRuntime.odoo_paused` so the - measure mirror (`harness/datastore.py`, which cannot import this layer) asks the SAME question - the customer pool does. This name stays because `routes_customers`, `routes_products` and - `verify_api` all call it — moving the logic without moving the door keeps one answer and - costs no caller a change. - """ - return bool(rt.odoo_paused()) - - -def _snap_scope_key(team_id, agent): - return f"t={team_id}|a={agent}" - - -def load_pool_snapshot(rt, team_id, agent): - """(ts, rows) from the persisted snapshot for this exact scope, or None. NEVER a wider - scope's rows — serving the consolidated snapshot to a scoped user would widen their book.""" - try: - snap = (rt.get(SNAPSHOT_KEY) or {}).get("odoo_pool") or {} - e = snap.get(_snap_scope_key(team_id, agent)) - if isinstance(e, dict) and isinstance(e.get("rows"), list): - return float(e.get("ts") or 0), e["rows"] - except Exception: - pass - return None - - -def save_pool_snapshots(rt, taken_by=""): - """Persist every currently-cached pool scope as the pause-time snapshot ('the last - successful sync', made concrete). Ensures the consolidated default scope exists first so - a pause on a cold process still captures something to serve.""" - import time as _time - import routes_customers as _rc - try: - _rc._pool_for(rt, None, None) # the scope every admin/all-BU account lands on - except Exception: - pass # cold + Odoo down: persist whatever IS cached - pools = {} - for key, entry in list(rt.pool_cache.items()): - if (isinstance(key, tuple) and len(key) == 3 and key[0] == "pool" - and isinstance(entry, tuple) and len(entry) == 2 - and isinstance(entry[1], list)): - pools[_snap_scope_key(key[1], key[2])] = {"ts": entry[0], "rows": entry[1]} - if not pools: - return 0 - - def _up(cur): - cur["odoo_pool"] = pools - cur["taken"] = _time.strftime("%Y-%m-%dT%H:%M:%S") - cur["takenBy"] = str(taken_by or "") - return cur - - rt.update(SNAPSHOT_KEY, _up, flush="sync") - return len(pools) - - -def _rel_reconnect(rt): - """Lift the W32-T16 freeze. Its own function so `add_key` and the reconnect route cannot - disagree about what "resume" means.""" - import odoo_relational as rel - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - cur["frozen"] = False - cur.pop("frozenAt", None) - cur.pop("frozenBy", None) - return cur - - rt.update(rel.CONFIG_KEY, _up, flush="sync") - return True - - -# ══════════════════════════════════════════════════ W35-T45 / R11: THE ENV -> KEYCHAIN MIGRATION -# -# R11: *"Tenant #0's Odoo credential MIGRATES onto the keychain, with the environment kept as -# fallback."* The owner chose this over a read-only display row WITH THE MIGRATION RISK STATED, so -# the risk is what this block is mostly about. -# -# ⭐ WHAT WAS ALREADY TRUE, CHECKED BEFORE ANY OF IT WAS WRITTEN: the keychain-first-then-env -# RESOLVER has existed since the 2026-08-04 cutover. `harness/runtime.py::odoo_source` is already -# (1) a keychain `odoo` entry, (2) tenant #0's compiled env connector, (3) None for anybody else, -# and `visible_entries` already serves a non-admin the row WITHOUT a preview. So R11 is not "build -# a resolver" — it is "give tenant #0 the ROW", which is the only reason its Keychain page looks -# empty while its Odoo grids work. -# -# ⛔⛔ AND THE ONE REAL HAZARD IS NOT THE CREDENTIAL, IT IS THE PAUSE FLAG. `odoo_flag_key()` returns -# the first keychain `odoo` entry id when one exists and `ENV_ODOO_FLAG_KEY` ("odoo-env") otherwise — -# so CREATING THE ENTRY MOVES THE ADDRESS THE PAUSE FLAG LIVES AT. A tenant #0 that was paused under -# `odoo-env` would come back UNPAUSED, silently, at the first boot after this ships: the connector -# resumes pulling live Odoo because a migration changed which key the freeze was stored under. That -# is D-259's exact shape (a pause written under one key and read under another) and -# [[a-guard-bound-to-a-role-stops-guarding-when-the-role-moves]]. The flag is carried across in the -# SAME pass, and the carry is asserted. - - -def _env_odoo_fields(): - """The four env values `odoo_client` reads, or `(None, why)` when they are not all present. - - ⛔ ALL FOUR OR NOTHING, and this completeness check is load-bearing rather than defensive. - Creating the entry makes `odoo_source` resolve through branch 1 INSTEAD of the env — so a - PARTIAL migration would hand the connector `{url, db}` and no key and take tenant #0's Odoo - offline, on a deployment where it had been working. The env fallback cannot save it, because the - entry's existence is what turns the fallback off. - ⚠ The names are `odoo_client.py`'s own (`ODOO_URL`/`ODOO_DB`/`ODOO_USER`/`ODOO_API_KEY`) and the - field names are `harness/connectors/odoo.py`'s stored shape (`{url, db, user, api_key}`). Two - vocabularies meet here; nowhere else. - """ - want = (("url", "ODOO_URL"), ("db", "ODOO_DB"), - ("user", "ODOO_USER"), ("api_key", "ODOO_API_KEY")) - got = {field: (os.environ.get(env) or "").strip() for field, env in want} - missing = sorted(env for field, env in want if not got[field]) - if missing: - return None, (f"the environment is missing {', '.join(missing)}, and a partial credential " - f"would take this tenant's Odoo offline rather than migrate it") - return got, "" - - -def migrate_env_odoo(rt): - """R11 — put tenant #0's environment Odoo credential on its keychain, once. Returns a report. - - `{"done": bool, "entry": id|"", "carried_pause": bool, "why": str}` — `why` is filled on every - path including the skips, because "already migrated", "no keychain key on this deployment" and - "the env is incomplete" are three different operator actions. - - ⛔⛔ IT MUST RUN IN THE CONTAINER, WHICH IS WHY `main.py` CALLS IT AND NO SCRIPT DOES. D-195, - measured three times: a developer's CLI write to the tenant store is reverted by the running - Space within a minute (download-modify-upload, last-write-wins) — and **the write reports success - every time**, then a fresh read confirms it, and it is gone by the next poll. A CLI migration here - would be a dry run that lies, and the thing it would lie about is a credential. - - ⚠ FAIL-QUIET AND IDEMPOTENT. It runs on EVERY boot; the second one must be a no-op and a - third-party failure must not take the boot down. - """ - out = {"done": False, "entry": "", "carried_pause": False, "why": ""} - if not env_odoo_available(rt): - # Not tenant #0, or this deployment has no env Odoo at all. Both are normal states. - out["why"] = "this tenant has no environment Odoo credential to migrate" - return out - fields, why = _env_odoo_fields() - if not fields: - out["why"] = why - return out - # ⛔ READ THE PAUSE FLAG BEFORE THE WRITE. After the entry exists, `odoo_flag_key()` answers the - # NEW key and the old one is unreachable through the resolver — so the only moment this fact can - # be observed is now. [[undo-capture-before-the-write]] applied to a guard rather than to data. - try: - was_paused = bool(((rt.get(_CONNECTOR_FLAGS_KEY) or {}) - .get(_ENV_ODOO_FLAG_KEY) or {}).get("paused")) - except Exception: # noqa: BLE001 - was_paused = False - row, why = _kc().ensure_entry_of_type( - rt, "odoo", "Odoo (migrated from this deployment)", fields, "system") - if row is None: - out["why"] = why - return out - out["done"], out["entry"] = True, row["id"] - if was_paused: - # The freeze followed the credential. Without this the connector silently RESUMES pulling - # live Odoo at the first boot after the migration. - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - entry = dict(cur.get(row["id"]) or {}) - entry["paused"] = True - entry["pausedBy"] = "system" - entry["pausedNote"] = ("carried over from the environment source when the credential " - "was migrated onto the keychain") - cur[row["id"]] = entry - return cur - - try: - rt.update(_CONNECTOR_FLAGS_KEY, _up, flush="sync") - out["carried_pause"] = True - except Exception as exc: # noqa: BLE001 - # ⛔ SAID OUT LOUD. A migration that moved the credential and lost the freeze is worse - # than one that did not run, so this is the one failure that must never be silent. - out["why"] = (f"the credential migrated but the PAUSE could not be carried over " - f"({type(exc).__name__}), so this tenant's Odoo is no longer frozen") - return out - - -def _own_row(session, entry_id): - """The visible row for `entry_id`, or a 404. ⛔ A 404 rather than a 403 for an entry the - caller cannot see: telling a member that somebody else's personal credential EXISTS is the - disclosure the scope is for.""" - row = next((e for e in visible_entries(session.runtime, session.uname, - bool(session.admin)) - if e["id"] == str(entry_id)), None) - if row is None: - raise err(404, "no_entry", "no such key") - return row - - -@router.get("/admin/keychain") -def list_keychain(session: Session = Depends(require_session)): - """⭐ W32-T11 / R4 — SESSION-GATED, NOT ADMIN-GATED, and that is the ruling not a relaxation. - R4 puts a PERSONAL connection in every member's hands, so a room only an admin can open would - ship the feature and no door to it. The wall moved INTO the payload: a member sees the - business-wide entries and their own, never anybody else's personal one.""" - kc = _kc() - return {"entries": visible_entries(session.runtime, session.uname, - bool(session.admin)), - "locked": not kc.unlocked(), - #: the vocabulary and the permission, so the client renders a picker it can honour - #: rather than offering an option the server will refuse (R4: business is admin-only). - "scopes": list(SCOPES), "canBusiness": bool(session.admin), - "tenantWideTypes": list(TENANT_WIDE_TYPES)} - - -@router.post("/admin/keychain", status_code=201) -def add_key(body: dict = Body(default=None), session: Session = Depends(require_session)): - kc = _kc() - body = body or {} - if not session.runtime.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable") - scope = clean_scope(body.get("scope")) - if scope is None: - raise err(400, "bad_scope", - f"scope must be one of {', '.join(SCOPES)}") - if scope == "business" and not session.admin: - raise err(403, "not_admin", - "a business-wide connection applies to everyone in this workspace, so only an " - "administrator can create one. You can add it as a personal connection instead.") - etype = str(body.get("type") or "").strip().lower() - if scope == "personal" and etype in TENANT_WIDE_TYPES: - # ⛔ REPORTED, NOT COERCED (W30/R6's second sentence). See TENANT_WIDE_TYPES above: this - # credential is what the WHOLE workspace's databases are built from, so "personal" would - # be a label on a tenant-wide key rather than a boundary around it. - raise err(400, "scope_not_available", - f"a {etype} connection is what this whole workspace's databases are read " - f"through, so it is always business-wide — it cannot be a personal connection. " - f"An administrator can add it for everyone.") - try: - row = kc.add_entry(session.runtime, body.get("label"), body.get("type"), - body.get("fields"), session.uname) - except kc.KeychainLocked as e: - raise err(503, "keychain_locked", - f"the keychain is locked — {e}. A secret is never stored unencrypted.") - except ValueError as e: - raise err(400, "bad_entry", str(e)) - except Exception: - raise err(503, "store_unavailable", "the entry was not saved — try again") - # ⛔⛔ A PERSONAL ENTRY THAT LOSES ITS SCOPE ROW READS AS BUSINESS-WIDE — i.e. the failure mode - # of a side bucket is to publish a credential, not to hide one. So the second write is not - # best-effort: if it does not land, the entry is REMOVED and the caller is told nothing was - # stored. `business` needs no row at all, so this branch is the only one that can be partial. - # ⭐ W32-T16 / R10's SECOND SENTENCE — *"Reconnecting resumes into the same tables."* Storing - # an Odoo credential IS reconnecting, so it lifts the freeze here rather than making the admin - # find a second switch. The tables were never dropped, so "resume" is one flag. - if etype == "odoo": - try: - _rel_reconnect(session.runtime) - except Exception: # noqa: BLE001 - pass - if scope != DEFAULT_SCOPE: - try: - _write_scope(session.runtime, row["id"], scope, session.uname) - except Exception: - try: - kc.delete_entry(session.runtime, row["id"]) - except Exception: # noqa: BLE001 - pass - raise err(503, "store_unavailable", - "the key was not saved — its sharing setting could not be stored, so " - "nothing was kept. Try again.") - return {"entry": {**row, "scope": scope, "owner": session.uname}} - - -@router.put("/admin/keychain/{entry_id}") -def update_key(entry_id: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Contract C1's scope door. Only the scope moves — a stored secret is never re-openable, so - "edit this key" means "replace it" and that is `DELETE` + `POST`.""" - row = _own_row(session, entry_id) - scope = clean_scope((body or {}).get("scope"), default=None) - if scope is None: - raise err(400, "bad_scope", f"scope must be one of {', '.join(SCOPES)}") - if scope == "business" and not session.admin: - raise err(403, "not_admin", - "a business-wide connection applies to everyone in this workspace, so only an " - "administrator can make one business-wide.") - if not _may_touch(session, row): - raise err(403, "not_yours", "this connection is not yours to change") - if scope == "personal" and str(row.get("type") or "") in TENANT_WIDE_TYPES: - raise err(400, "scope_not_available", - f"a {row.get('type')} connection is what this whole workspace's databases are " - f"read through, so it is always business-wide.") - owner = row.get("owner") or session.uname - try: - _write_scope(session.runtime, entry_id, scope, owner) - except Exception: - raise err(503, "store_unavailable", "the change was not saved — try again") - return {"entry": {**row, "scope": scope, "owner": owner if scope == "personal" else ""}} - - -@router.delete("/admin/keychain/{entry_id}") -def delete_key(entry_id: str, session: Session = Depends(require_session)): - row = _own_row(session, entry_id) - if not _may_touch(session, row): - raise err(403, "not_yours", "this connection is not yours to delete") - try: - _kc().delete_entry(session.runtime, entry_id) - _write_scope(session.runtime, entry_id, DEFAULT_SCOPE, "") # drop the side row with it - except Exception: - raise err(503, "store_unavailable", "the delete did not land — try again") - return {"ok": True} - - -@router.post("/admin/keychain/{entry_id}/test") -def test_key(entry_id: str, session: Session = Depends(require_session)): - _own_row(session, entry_id) # 404 for an entry this caller may not see - return _kc().test_entry(session.runtime, entry_id) - - -def _unsynced_customer_records(session): - """R3's guardrail, tenant #0's customer topic: overlay-holding pids the CURRENT pool no - longer serves. Overlays are unioned across EVERY user of the table (the guardrail is a - tenant fact, not a per-user one). Honest degradation: when the pool cannot be built the - answer is `known: False`, never a fabricated zero.""" - try: - import core.table_store as table_store - bucket = session.runtime.get("customer_table_workspace") or {} - overlay_pids = {} - for uname, ws in bucket.items(): - if uname == table_store.SHARED_KEY or not isinstance(ws, dict): - continue - for pid, cells in (ws.get("overlays") or {}).items(): - if isinstance(cells, dict) and cells: - overlay_pids.setdefault(str(pid), cells) - if not overlay_pids: - return {"known": True, "count": 0, "rows": []} - from routes_customers import allowed_pids - pool = {str(p) for p in allowed_pids(session)} - orphans = sorted((p for p in overlay_pids if p not in pool), key=lambda x: int(x) - if str(x).isdigit() else 0) - rows = [] - for p in orphans[:50]: - cells = overlay_pids[p] - hint = next((str(v) for v in cells.values() if str(v).strip()), "") - rows.append({"pid": int(p) if str(p).isdigit() else p, - "fields": len(cells), "hint": hint[:80]}) - return {"known": True, "count": len(orphans), "rows": rows, - "shown": min(len(orphans), 50)} - except Exception as e: - return {"known": False, "count": None, "rows": [], - "note": f"pool unavailable — {type(e).__name__}"} - - -@router.get("/admin/connectors") -def connectors(session: Session = Depends(require_session)): - kc = _kc() - flags = session.runtime.get(_CONNECTOR_FLAGS_KEY) or {} - # ⭐ W32-T11 / R4 — THE SAME VISIBILITY RULE AS THE KEYCHAIN, because this pane is the same - # facts with a status column. Reading `kc.list_entries` here instead would have shown a member - # every colleague's personal connection on the screen next door to the one that hides them. - entries = visible_entries(session.runtime, session.uname, bool(session.admin)) - # R3 cutover (2026-08-04): which source would actually serve this tenant's Odoo queries — - # mirrors TenantRuntime.odoo_source() exactly: first unlocked keychain odoo entry, else env - # for tenant #0 only, else nothing (fail closed — never another tenant's environment). - # ⚠ W32-T11: computed over the TENANT's entries, not over `entries` above. "Which source - # serves this workspace" is one fact for everybody, and deriving it from a per-USER list would - # make the answer depend on who opened the pane. Personal entries are excluded for the same - # reason `TENANT_WIDE_TYPES` refuses them: they must never become the workspace's source. - _scopes = _scope_rows(session.runtime) - first_odoo = next((e["id"] for e in kc.list_entries(session.runtime) - if e["type"] == "odoo" - and entry_scope(session.runtime, e["id"], _scopes)[0] != "personal"), None) - # ⛔ W32-T10 — the env leg goes through `env_odoo_available` now, so this route and the - # connectors DIRECTORY answer "does the environment serve this tenant?" with one function - # instead of two spellings that agreed until a second tenant signed in. - if first_odoo and kc.unlocked(): - resolved = f"keychain:{first_odoo}" - elif env_odoo_available(session.runtime): - resolved = "env" - else: - resolved = None - rows = [] - if env_odoo_available(session.runtime): - rows.append({"key": _ENV_ODOO_FLAG_KEY, "label": "Odoo (environment)", "type": "odoo", - "source": "env", "active": resolved == "env", - # the deployment's own credential — business-wide by construction, and it - # has no owner to be personal to. - "scope": DEFAULT_SCOPE, "owner": "", - "paused": bool((flags.get(_ENV_ODOO_FLAG_KEY) or {}).get("paused"))}) - for e in entries: - rows.append({"key": e["id"], "label": e["label"], "type": e["type"], - "source": "keychain", "preview": e["preview"], - "scope": e.get("scope") or DEFAULT_SCOPE, "owner": e.get("owner") or "", - "active": (e["type"] == "odoo" and resolved == f"keychain:{e['id']}"), - "paused": bool((flags.get(e["id"]) or {}).get("paused"))}) - out = {"connectors": rows, "locked": not kc.unlocked(), "resolved": resolved, - "scopes": list(SCOPES), "canBusiness": bool(session.admin), - # ⭐ D-10 (wave 24) — THIS SENTENCE IS NOW TRUE OF EVERY PATH, which it was not before. - # DEBT-2 (2026-08-04) froze the CUSTOMER pool and this note honestly disclosed the - # hole it left: "measures not already computed may still reach the source". D-10 - # closed that hole — `harness/datastore.py` (the mirror every measure column is - # answered from) refuses to sync while paused, and `routes_products._pool_for` got the - # guard its customer sibling has had since DEBT-2. So the caveat is deleted rather - # than left standing, because a warning that outlives its defect teaches the reader to - # ignore warnings. - # ⚠ THE THREE BEHAVIOURS ARE NAMED SEPARATELY on purpose: they are genuinely - # different answers (a persisted snapshot, an in-process cache, a frozen mirror), and - # collapsing them into "everything freezes" would be the kind of tidy summary that - # stops being true the first time one of them changes. - # ⭐ D-62 CLOSED (wave 27) — AND THE REGISTER'S DIAGNOSIS OF IT WAS WRONG, so the - # correction is recorded here rather than silently applied. D-62 said this note - # "promises a behaviour on a dashboard measure path that has been dead since W16". - # MEASURED 2026-08-08, and it is not: measure COLUMNS are grid columns answered from - # the DuckDB mirror, and `harness/datastore.py` genuinely refuses to sync while - # paused (`source_paused()` at four sites), so that clause was TRUE. The dead path is - # `/api/v1/pages/{key}` (D-52), which this note never mentioned. - # - # ⛔ THE REAL DEFECT WAS THE OPPOSITE ONE, and it was the last sentence: "so figures - # stop moving rather than going blank". BOTH pool paths answer **503** when they have - # no copy to serve — the customer path for a scope with no snapshot - # (`routes_customers.py:88`) and the product path ALWAYS after a restart, because - # there is no product snapshot bucket at all (`routes_products.py:60-73`, which says - # so in as many words). So a paused connector plus a restarted server is exactly the - # blank screen this sentence promised could not happen. A warning that over-promises - # is worse than none: it is the sentence somebody quotes when the screen disagrees. - "pausedNote": ("Pausing a connector never deletes data — notes, custom fields and " - "views stay, and nothing reaches the source while it is paused. " - "Anything this server has already read keeps showing: the customer " - "workspace serves its pause-time snapshot, the product list serves " - "the last copy read since startup, and measure columns keep answering " - "from the mirror as it stood when you paused. What has NOT been read " - "cannot be shown — a scope with no snapshot, or the product list after " - "a restart, reports that the source is paused instead of showing " - "figures. Resume to start reading live again.")} - # ⛔⛔ W32-T11 — ADMIN-GATED, AND THIS IS A DISCLOSURE FIX, NOT TIDINESS. Opening this route to - # members (R4) opened this block with it, and `_unsynced_customer_records` is the one thing on - # the payload that is NOT about connectors: it unions overlays across EVERY user of the - # customer table — its own docstring says so, *"the guardrail is a tenant fact, not a per-user - # one"* — and returns `hint`, the first non-empty cell of somebody else's overlay. So a member - # would have read colleagues' typed notes off the Connectors pane. It also filters against - # `allowed_pids(session)`, so a BU-scoped member's narrower pool inflates the orphan count and - # the number itself becomes wrong for them as well as private. - # ⚠ The lesson generalises past this line: opening a route widens EVERY field it already - # returned, and the audit has to walk the payload, not the entry list I was thinking about. - if session.tenant == "royal-imports" and session.admin: - out["unsynced"] = _unsynced_customer_records(session) - return out - - -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# ⭐⭐ W32-T15/T16/T17 / CONTRACT C2 / RULINGS R9, R10, R11 — THE ODOO CONNECTOR ACTUALLY OPENS -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# -# The owner clicked "Manage keys" on Odoo and found a credential list. Not: which server database -# this workspace reads, which of the ten mirrored grids it wants, how often they sync, or how to -# stop. Every decision below lives in `odoo_relational` (the module `refresh()` reads) so that a -# switch flipped here is a switch the sync path obeys — a config the route knows and the sync -# path does not is a control that does nothing and reports success. -def _rel(): - import odoo_relational as rel - return rel - - -def _odoo_entry(session): - """The keychain entry SERVING this tenant's Odoo, or None when the environment is (or nothing - is). Business-scoped by construction — `TENANT_WIDE_TYPES` refuses a personal one.""" - kc = _kc() - scopes = _scope_rows(session.runtime) - for e in kc.list_entries(session.runtime): - if e["type"] == "odoo" and entry_scope(session.runtime, e["id"], scopes)[0] != "personal": - return e - return None - - -def _odoo_source_fields(session, entry): - """`(serverDb, serverUrl, apiUser, editable)` — what the panel may SHOW about the connection. - - ⛔ NEVER THE SECRET. `read_fields` is documented as the connector layer's internal and no - route returns its output; this returns the three fields that identify WHICH server, and the - api key is not among them. The masked preview is the entry's own and was computed at write. - ⚠ The ENVIRONMENT source is not editable and says so: it is the deployment's `.env`, shared by - the process, and an admin editing it from a tenant screen would be editing the container. - """ - if entry is None: - return (os.environ.get("ODOO_DB", ""), os.environ.get("ODOO_URL", ""), - os.environ.get("ODOO_USER", ""), False) - try: - f = _kc().read_fields(session.runtime, entry["id"]) or {} - except Exception: # noqa: BLE001 - return ("", "", "", True) # locked keychain: honest blanks, still editable - return (str(f.get("db") or ""), str(f.get("url") or ""), str(f.get("user") or ""), True) - - -def _odoo_admin(session): - """C2's doors are admin doors: they show the credential that serves EVERYONE and can turn the - whole workspace's databases off. R4's personal scope has nothing to say here — a tenant-wide - type cannot be personal in the first place.""" - if not session.admin: - raise err(403, "not_admin", - "the Odoo connection serves this whole workspace, so only an administrator can " - "configure it") - - -@router.get("/admin/connectors/odoo/config") -def odoo_config(session: Session = Depends(require_session)): - """Contract C2's read: `{serverDb, grids, syncEvery, canDisconnect}` and the rest of what a - person needs to see before changing any of it.""" - _odoo_admin(session) - rel = _rel() - entry = _odoo_entry(session) - server_db, server_url, api_user, editable = _odoo_source_fields(session, entry) - cfg = rel.read_config(session.runtime) - return { - "applicable": bool(rel.is_royal(session.tenant)), - "source": "keychain" if entry else ("env" if env_odoo_available(session.runtime) - else "none"), - "entryId": (entry or {}).get("id", ""), - "label": (entry or {}).get("label", "Odoo (environment)"), - "preview": (entry or {}).get("preview", ""), - "serverDb": server_db, "serverUrl": server_url, "apiUser": api_user, - "serverDbEditable": editable, - "grids": rel.grid_choices(session.runtime), - "syncEvery": cfg["syncEvery"], - "syncOptions": list(rel.SYNC_PRESETS), - "syncFloorSeconds": rel.SYNC_FLOOR_SECONDS, - "frozen": cfg["frozen"], "frozenAt": cfg["frozenAt"], - # ⚠ There is nothing to disconnect FROM when the source is the deployment environment: - # tenant #0's `.env` is not this tenant's to remove. Said as a field so the client renders - # no button rather than one that 400s. - "canDisconnect": bool(entry) or (env_odoo_available(session.runtime) - and not cfg["frozen"]), - } - - -@router.put("/admin/connectors/odoo/config") -def odoo_config_put(body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Contract C2's write. Grids, cadence and the server database — each optional, each REPORTED - back rather than silently applied.""" - _odoo_admin(session) - rel = _rel() - body = body or {} - notes = [] - - grids = body.get("grids") - known = {c["key"] for c in rel.grid_choices(session.runtime)} - clean_grids = None - if isinstance(grids, dict): - unknown = sorted(str(k) for k in grids if str(k) not in known) - if unknown: - # ⛔ NAMED, NOT DROPPED. A key we do not serve is a client that believes in a grid - # this connector does not have, and swallowing it makes the two disagree quietly. - raise err(400, "unknown_grid", - f"this connector has no grid called {', '.join(unknown)}") - clean_grids = {str(k): bool(v) for k, v in grids.items()} - if clean_grids and not any(clean_grids.get(k, True) for k in known): - notes.append("every grid is switched off — nothing will be materialised on the next " - "sync, and the databases you already have are left untouched") - - every = body.get("syncEvery") - clean_every = None - if every is not None: - clean_every = str(every).strip().lower() - if clean_every not in rel.SYNC_PRESETS: - # ⛔⛔ R11 + W30/R6's SECOND SENTENCE: the floor is enforced AND the caller is told. - # A crafted `"5m"` is CLAMPED to the floor and the response says so — never applied, - # and never silently ignored either, because a control that discards your answer - # without a word is how a limit becomes invisible. - clean_every = rel.DEFAULT_SYNC - notes.append(f"{every!r} is not an interval this connector offers, and anything under " - f"{rel.SYNC_FLOOR_SECONDS // 60} minutes is not available at all — the " - f"sync interval was set to the {rel.DEFAULT_SYNC} floor instead") - - server_db = body.get("serverDb") - if server_db is not None: - server_db = " ".join(str(server_db).split())[:80] - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - if clean_grids is not None: - cur.setdefault("grids", {}).update(clean_grids) - if clean_every is not None: - cur["syncEvery"] = clean_every - return cur - - try: - session.runtime.update(rel.CONFIG_KEY, _up, flush="sync") - except Exception: - raise err(503, "store_unavailable", "the change was not saved — try again") - - if server_db: - notes.append(_rewrite_server_db(session, server_db)) - - # ⭐⭐ W33-T65 / W30-R6's SECOND SENTENCE — THE CADENCE IS SET AND ONLY PARTLY OBEYED, AND THE - # PERSON SETTING IT IS THE ONE WHO HAS TO BE TOLD. Measured, not guessed: - # · `main.py::_store_resync_loop` reads the interval from `sync_seconds(get_runtime( - # "royal-imports"))` — a HARDCODED slug — and then sleeps ONCE for the whole process. So - # for tenant #0 this control moves EVERYBODY's sync, and for every other tenant the value - # is stored, clamped, displayed and never read. - # · `manual` stores and reads back as `None`, and the loop has no branch on it: it sleeps a - # default 1800 s and syncs anyway. "Only when I ask" asks all the same. - # ⛔ NEITHER IS FIXABLE FROM THIS FILE — the loop lives in `main.py`, which this lane does not - # own — and shipping a setting that silently does nothing is the exact failure R6 names. So it - # is REPORTED here, at the moment of the change, with what it really controls. Delete these - # notes when the loop becomes per-tenant, not before. - if clean_every is not None: - if not rel.is_royal(session.tenant): - notes.append("this interval is saved, but the sync loop currently reads its schedule " - "from one workspace for the whole deployment — so it will not change how " - "often YOUR data refreshes until per-workspace scheduling ships") - else: - notes.append("this interval is saved and it is the one the deployment's sync loop " - "uses — it changes the refresh rate for every workspace on this " - "deployment, not only this one") - if clean_every == "manual": - notes.append("⚠ 'manual' does not yet stop the background sync: the loop has no " - "manual-only branch, so data still refreshes on the default interval") - - out = odoo_config(session) - return {**out, "notes": [n for n in notes if n]} - - -def _rewrite_server_db(session, server_db): - """Point the stored Odoo credential at a different server database (R9's first reading). - - ⛔ THERE IS NO "UPDATE ENTRY" IN THE KEYCHAIN, and writing one here would be a SECOND copy of - how a secret is encrypted and previewed — the thing `core/keychain.py` exists to hold alone. - So this is add-then-delete through the module's own doors, with the side rows (pause flag, - scope) carried across because they are keyed by ENTRY ID. - ⚠ THE ORDER IS DELIBERATE AND THE WINDOW IS REAL: for the moment between the add and the - delete this tenant has TWO odoo entries, and `odoo_creds` takes the first by id sort — so a - resync landing inside that window could read the OLD database. The alternative order can - leave the workspace with no credential at all, which is worse than one stale read. Milliseconds - of ambiguity beats a lost key. - """ - kc = _kc() - entry = _odoo_entry(session) - if entry is None: - return ("the server database is set on this deployment's environment, not in the " - "keychain, so it was not changed here") - try: - fields = kc.read_fields(session.runtime, entry["id"]) or {} - except kc.KeychainLocked as e: - raise err(503, "keychain_locked", f"the keychain is locked — {e}") - if not fields: - raise err(400, "bad_entry", "this credential could not be read back to be changed") - if str(fields.get("db") or "") == server_db: - return "" - fields["db"] = server_db - try: - new = kc.add_entry(session.runtime, entry["label"], "odoo", fields, session.uname) - except Exception: - raise err(503, "store_unavailable", - "the server database was not changed — the existing connection is untouched") - # carry the side rows across, then retire the old entry - try: - flags = session.runtime.get(_CONNECTOR_FLAGS_KEY) or {} - if entry["id"] in flags: - def _mv(cur): - cur[new["id"]] = cur.pop(entry["id"], {}) - return cur - session.runtime.update(_CONNECTOR_FLAGS_KEY, _mv) - kc.delete_entry(session.runtime, entry["id"]) - _write_scope(session.runtime, entry["id"], DEFAULT_SCOPE, "") - except Exception: # noqa: BLE001 - return (f"the connection now points at {server_db}, but the previous credential could " - f"not be removed — delete it under Keychains") - return f"the connection now points at the {server_db} database" - - -@router.post("/admin/connectors/odoo/disconnect") -def odoo_disconnect(session: Session = Depends(require_session)): - """R10 — remove the credential and FREEZE the grids as static data. - - ⛔ DISTINCT FROM PAUSE, and the difference is the credential. Pause is temporary and keeps the - key; disconnect deletes it and marks the databases frozen so nothing refreshes them again — - including the boot rebuild and the resync loop, which for tenant #0 would otherwise - re-materialise from the process ENVIRONMENT and quietly undo the disconnect. - ⛔⛔ AND IT DELETES NOTHING ELSE. The owner's words are *"so we don't fuck up"*: every row and - every FIELD DEFINITION stays, user-added columns included, because a field a person added is - the thing a naive freeze drops first. This route never touches `fields` or `rows` — it writes - one flag in a different bucket, which is what makes that guarantee structural rather than - careful. - """ - _odoo_admin(session) - rel = _rel() - import datetime as _dt - entry = _odoo_entry(session) - if not entry and not env_odoo_available(session.runtime): - raise err(400, "not_connected", "this workspace has no Odoo connection to disconnect") - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - cur["frozen"] = True - cur["frozenAt"] = _dt.datetime.now().strftime("%Y-%m-%dT%H:%M:%S") - cur["frozenBy"] = str(session.uname) - return cur - - # ⭐⭐ W33-T65 — THE FREEZE HAD A HOLE, AND THE NOTE BELOW WAS THE THING THAT MADE IT A DEFECT - # RATHER THAN A LIMIT. `rel.frozen` has exactly ONE consumer, `odoo_relational.refresh`, which - # materialises the EIGHT copied grids. The other two (`ut_odoo_order_lines`, - # `ut_odoo_gl_lines` — `READ_THROUGH_KEYS`) do not go through `refresh` at all: they read - # THROUGH the per-tenant DuckDB mirror, and the mirror is advanced by `datastore.sync_all`, - # which gates on the connector PAUSE flag and has never heard of `frozen`. So a disconnected - # workspace kept serving LIVE, still-moving rows in its two biggest grids while this route's - # own sentence promised *"nothing is being refreshed"*. - # - # ⛔ THE FIX IS TO MAKE THE SENTENCE TRUE, not to soften it. Disconnect now flips the pause - # flag on the RESOLVED source as well, which is the switch `sync_all` and `reconcile_deletes` - # actually read — so both halves of "frozen" mean the same thing. Two orderings are borrowed - # from `pause_connector` because it learned them the hard way: - # · the flag key is resolved BEFORE the credential is deleted — after the delete there is no - # resolved source left to name, and the flag would land under a key nothing reads (D-10). - # · the snapshot is captured BEFORE the flag flips, so there is a last-successful-sync to - # serve; a failed capture leaves the connector live rather than paused-with-nothing. - _, flag_key = _resolved_odoo_key(session.runtime) - snapshots = 0 - if flag_key: - try: - snapshots = save_pool_snapshots(session.runtime, taken_by=session.uname) - except Exception: # noqa: BLE001 - # A snapshot is a nicety; the freeze is the promise. Reported, never fatal. - snapshots = 0 - - # ⚠ THE FLAG FIRST, THE CREDENTIAL SECOND. If the flag write fails, nothing has happened and - # the connector is still live; if the delete failed AFTER the flag landed, the tenant is - # frozen with an unused key, which is recoverable from the Keychains pane. The reverse order - # can leave a workspace with no key and a connector that still tries to sync. - try: - session.runtime.update(rel.CONFIG_KEY, _up, flush="sync") - except Exception: - raise err(503, "store_unavailable", "nothing was disconnected — try again") - - paused_mirror = False - if flag_key: - def _pause(cur): - cur = cur if isinstance(cur, dict) else {} - cur[str(flag_key)] = {"paused": True} - return cur - try: - session.runtime.update(_CONNECTOR_FLAGS_KEY, _pause, flush="sync") - paused_mirror = True - except Exception: # noqa: BLE001 - paused_mirror = False - - removed = "" - if entry is not None: - try: - _kc().delete_entry(session.runtime, entry["id"]) - _write_scope(session.runtime, entry["id"], DEFAULT_SCOPE, "") - removed = entry["id"] - except Exception: - raise err(503, "store_unavailable", - "the databases are frozen but the stored credential was not removed — " - "delete it under Keychains") - - # ⛔ THE SENTENCE IS COMPOSED, NOT CONSTANT, because the two sources genuinely differ and the - # old fixed string was wrong about one of them. A tenant whose Odoo came from the DEPLOYMENT - # ENVIRONMENT has no credential for this route to remove — `.env` is the container's, not a - # tenant screen's — so it said "the key back" about a key it never held. R6's second sentence: - # the limit that cannot be removed is REPORTED, with what to do instead. - note = ("Your Odoo databases are frozen: every row and every column you had is still there and " - "still readable, and nothing is being refreshed.") - if not paused_mirror and flag_key: - note += (" ⚠ The live mirror could not be paused, so the two read-through databases " - "(order lines and GL lines) may keep advancing — pause the Odoo connector under " - "Keychains to stop them.") - note += (" Reconnecting adds the key back and resumes into the same databases." if entry - else " This workspace's Odoo credential comes from the deployment environment, so " - "there was no stored key to remove — the databases are frozen and Reconnect " - "resumes them into the same tables.") - return {"frozen": True, "removedEntry": removed, - # ⚠ ON THE WIRE, so the client and a gate can both see which half happened. A boolean - # nobody returns is a guarantee nobody can check. - "pausedMirror": paused_mirror, "snapshots": snapshots, - "source": "keychain" if entry else "env", - "note": note} - - -@router.post("/admin/connectors/odoo/reconnect") -def odoo_reconnect(session: Session = Depends(require_session)): - """R10's second sentence — *"Reconnecting resumes into the same tables."* - - It clears the freeze and nothing else: the tables were never dropped, so there is nothing to - recreate. A tenant whose source was a keychain entry adds it back under Keychains first; this - is the switch that lets the sync path see it again. - """ - _odoo_admin(session) - try: - _rel_reconnect(session.runtime) - except Exception: - raise err(503, "store_unavailable", "the change was not saved — try again") - # ⭐⭐ W33-T65 — AND THE PAUSE DISCONNECT SET, or the freeze would be one-way. Clearing only - # `frozen` restores the eight materialised grids and leaves the mirror pinned forever, so the - # two read-through grids would sit at the disconnect date while the panel said "connected - # again" — the same disagreement between the halves of "frozen", pointing the other way. - # ⚠ Resolved AFTER `_rel_reconnect`: a tenant reconnects by adding the key back FIRST, so the - # resolved source only exists again by this point. - _, flag_key = _resolved_odoo_key(session.runtime) - resumed = False - if flag_key: - def _unpause(cur): - cur = cur if isinstance(cur, dict) else {} - cur[str(flag_key)] = {"paused": False} - return cur - try: - session.runtime.update(_CONNECTOR_FLAGS_KEY, _unpause, flush="sync") - resumed = True - except Exception: # noqa: BLE001 - resumed = False - connected = bool(_odoo_entry(session)) or env_odoo_available(session.runtime) - return {"frozen": False, "connected": connected, "resumedMirror": resumed, - "note": ("Odoo is connected again and the databases you already had will refresh in " - "place." if connected else - "The freeze is lifted, but there is no Odoo credential yet — add one under " - "Keychains and the databases resume into the same tables.")} - - -@router.post("/admin/connectors/{key}/pause") -def pause_connector(key: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - paused = bool((body or {}).get("paused")) - if not session.runtime.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable") - # ⭐ W32-T11 / R4 — pausing a BUSINESS-WIDE connector stops it for everyone, so it stays an - # admin act; pausing your own personal one is yours. The env source has no keychain row and - # is business-wide by construction, hence the admin fallthrough. - row = next((e for e in visible_entries(session.runtime, session.uname, - bool(session.admin)) - if e["id"] == str(key)), None) - if row is not None: - if not _may_touch(session, row): - raise err(403, "not_yours", "this connection is not yours to pause") - elif not session.admin: - raise err(403, "forbidden", "administrators only") - - # DEBT-2: pausing the RESOLVED Odoo source captures the snapshot FIRST, so there is a - # "last successful sync" to serve before the freeze takes effect. Capturing before the - # flag flips means a failed capture leaves the connector live (never paused-with-nothing). - snapshots = 0 - _, flag_key = _resolved_odoo_key(session.runtime) - if paused and flag_key and str(key) == flag_key: - snapshots = save_pool_snapshots(session.runtime, taken_by=session.uname) - - def _up(cur): - cur[str(key)] = {"paused": paused} - return cur - - try: - session.runtime.update(_CONNECTOR_FLAGS_KEY, _up) - except Exception: - raise err(503, "store_unavailable", "the change was not saved — try again") - return {"key": key, "paused": paused, "snapshots": snapshots} +"""routes_keychain.py — Keychains + Connectors admin surfaces (wave 18, C7 / R3). + +Keychain: encrypted per-tenant credential entries (`core/keychain.py`). The routes NEVER +return a decrypted field — list rows carry a masked preview, and the decrypt function is a +connector-layer internal. Connectors: the tenant's data sources as STATUS rows — Royal's +env-configured Odoo, keychain-held sources — plus R3's guardrail: the **Unsynced records** +count (rows holding overlay data whose pids the current pool no longer serves; counted and +drillable, never silently dropped) and a pause toggle whose v1 semantics are stated honestly +in the payload (`pausedNote`): pausing marks intent and warns; the source cutover ships with +the keychain cutover wave R3 staged. +""" +import os + +from fastapi import Body, Depends +from fastapi import APIRouter + +from deps import Session, err, require_session +# ⚠ W32-T11 / R4: `admin_gate` is GONE from this module, and its absence is the ruling. Every door +# here was admin-only, which made "a member may hold a personal connection" unbuildable — the wall +# moved from the ROUTE into the ROW (`may_see` / `_may_touch`), where a scope can be enforced per +# entry instead of per endpoint. `session.admin` is still what business-wide requires. + +router = APIRouter(prefix="/api/v1") + +#: ⭐ D-10 (wave 24): ONE literal, owned by the harness. It was spelled here AND implied by +#: `harness/runtime.py`'s reader; a pause flag written under one spelling and read under another +#: freezes nothing while reporting success, which is the shape of the bug D-10 books. +from harness.runtime import (CONNECTOR_FLAGS_KEY as _CONNECTOR_FLAGS_KEY, # noqa: E402 + ENV_ODOO_FLAG_KEY as _ENV_ODOO_FLAG_KEY) +#: DEBT-2 (2026-08-04): the last-successful-sync snapshot bucket. Written when the RESOLVED +#: odoo connector is paused; read by the pool path while paused; survives a Space restart. +SNAPSHOT_KEY = "connector_snapshots" + +#: ⛔⛔ W32-T10 / OWNER ITEM 6 — THE TENANT THAT OWNS THE PROCESS ENVIRONMENT. +#: +#: `ODOO_URL` and its siblings are tenant #0's `.env` / Space secrets, and ONE Space process serves +#: EVERY tenant. So "the environment has Odoo credentials" is a fact about this DEPLOYMENT, and +#: turning it into "your workspace is connected to Odoo" is only true for one slug. That is R3's +#: rule, stated in `harness/runtime.py::odoo_source` as *"NEVER the environment: env is tenant #0's +#: connection, and handing it to another tenant is the leak this method exists to prevent."* +#: +#: ⚠ IT IS SPELLED HERE BECAUSE THE RUNTIME EXPORTS NO CONSTANT FOR IT — `odoo_source` and +#: `odoo_flag_key` both carry the literal. `verify_meta`'s W32-T10 section asserts this value +#: against `runtime.py`'s own text, so the day the runtime's answer changes and this one does not, +#: the gate reds instead of the product quietly disagreeing with itself. +ENV_ODOO_TENANT = "royal-imports" + + +def env_odoo_available(rt): + """Does the PROCESS ENVIRONMENT offer an Odoo connection to THIS tenant? (W32-T10.) + + ⛔ THE ONE NORMALIZER FOR ONE QUESTION, and it exists because there were two answers to it. + `routes_connectors.directory._odoo` read a bare `os.environ.get("ODOO_URL")` with **no tenant + guard** — one process, every tenant — so a nurilab admin opening Connectors was told Odoo was + `connected` and offered "Manage keys" for a credential belonging to another company. This + module asked the same question correctly two functions below, which is the whole shape of + [[one-question-two-normalizers]]: the correct copy hides the wrong one until somebody signs in + as the second tenant. + + ⚠ NOT the same question as `rt.odoo_flag_key() == ENV_ODOO_FLAG_KEY`. That one answers *which + source WINS*, so it goes False the moment a keychain entry exists — correct for a pause flag, + wrong for "should the environment row be listed at all", which is what the connectors pane + needs in order to show an inactive env source beside an active keychain one. + + Never raises: a runtime that cannot answer is not connected. Fail closed — a missing guard is + how another tenant's environment got reported as this tenant's connection in the first place. + """ + try: + if not (os.environ.get("ODOO_URL") or "").strip(): + return False + return str(getattr(rt, "key", "") or "") == ENV_ODOO_TENANT + except Exception: # noqa: BLE001 + return False + + +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ W32-T11 / CONTRACT C1 / OWNER RULING R4 — BUSINESS-WIDE vs PERSONAL, ON EVERY CONNECTION +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# +# R4, verbatim: *"The business-wide vs personal split lands on ALL connections, not just Odoo. +# Every keychain entry and connector carries a scope, selectable per connection; business-wide is +# admin-only and applies to every user in the tenant."* +# +# ⛔ THE VOCABULARY IS DECLARED HERE AND NOWHERE ELSE (contract C1, and it names this file +# explicitly). `core/keychain.py` is the ENCRYPTED STORE and belongs to the integrator; a scope is +# an access rule, not a secret, so it lives in the layer that already decides who may ask. +# `verify_meta` asserts this tuple against `connectors/ConnectorsPage.tsx`, so the two halves +# cannot drift into two vocabularies. +SCOPES = ("business", "personal") +#: ⚠ THE READ DEFAULT, AND IT IS A READ DEFAULT — never a write (C1). Every entry stored before +#: this wave was, by construction, an admin's tenant-wide credential, so `business` is not a guess. +#: Materialising it would be a migration nobody authorised and would rewrite the store on the next +#: read [[a-migration-that-runs-on-the-next-write]]. +DEFAULT_SCOPE = "business" +#: The side bucket: `{entry_id: {"scope": "personal", "owner": ""}}`. +#: ⛔ A BUSINESS ENTRY WRITES NO ROW — it IS the default, so an absent row and a `business` row mean +#: the same thing and there is only one way to spell the common case. +SCOPE_KEY = "keychain_scopes" + +#: ⛔⛔ THE TYPES A WHOLE WORKSPACE READS THROUGH, WHICH THEREFORE CANNOT BE PERSONAL. +#: +#: `keychain.odoo_creds` and `keychain.meta_creds` both resolve to *the first entry of that type* +#: for the TENANT — that is what spawns `ut_odoo_*` / `ut_meta_*` and what every measure column is +#: answered from. So a member storing a personal Odoo key would not get "their own Odoo": they +#: would silently become the credential the entire workspace's databases are built from, which is +#: a credential elevation wearing a scope picker. +#: ⚠ REFUSED WITH A REASON, NEVER SILENTLY COERCED TO `business` — a picker that quietly changes +#: your answer is worse than one that says no (W30/R6's second sentence). The resolver itself is +#: `harness/runtime.py` / `core/keychain.py`, the integrator's files; this refusal closes the door +#: from the only side B owns, and the resolver-side guard is booked for A. +TENANT_WIDE_TYPES = ("odoo", "meta_ads") + + +def clean_scope(raw, default=DEFAULT_SCOPE): + """A scope word from the wire, or None when the caller said something we do not speak. + + Distinguishing "said nothing" (⇒ the default) from "said nonsense" (⇒ 400) is the whole + reason this returns None rather than falling back: a typo'd `"personel"` silently becoming + business-wide is exactly the failure a scope picker exists to prevent. + """ + if raw is None or (isinstance(raw, str) and not raw.strip()): + return default + got = str(raw).strip().lower() + return got if got in SCOPES else None + + +def _scope_rows(rt): + try: + return dict(rt.get(SCOPE_KEY) or {}) + except Exception: # noqa: BLE001 + return {} + + +def entry_scope(rt, entry_id, rows=None): + """`(scope, owner)` for one entry. `rows` is the bucket, passed in when walking a list so a + census does not re-read the store once per entry.""" + r = (rows if rows is not None else _scope_rows(rt)).get(str(entry_id)) + if not isinstance(r, dict): + return DEFAULT_SCOPE, "" + return (str(r.get("scope") or DEFAULT_SCOPE), str(r.get("owner") or "")) + + +def may_see(scope, owner, uname): + """R4's visibility rule: business-wide is everyone's, personal is its owner's. + + ⛔ AND AN ADMIN IS NOT AN EXCEPTION. R4 says a personal entry belongs to a person; the + per-user OAuth slots one module down have made the same call since wave 22 (*"a refresh token + is identity, not infrastructure"*). An admin who could read every member's personal credential + would make "personal" a label rather than a boundary. + """ + return scope != "personal" or str(owner) == str(uname) + + +def visible_entries(rt, uname, is_admin=False): + """This USER's view of the keychain: every business entry plus their own personal ones, each + row carrying its `scope` and `owner` so no caller has to ask a second time. + + ⛔ THE MASKED PREVIEW IS NOT PART OF "VISIBLE". R4 opens this room to members so they can + hold a connection of their own; it does not hand them four characters of the workspace's Odoo + key. So a business row a member did not create arrives WITHOUT `preview` — they can see that + the connection exists and is theirs to use, which is the whole of what R4 grants. The default + is the RESTRICTED one deliberately: a caller that forgets the argument leaks nothing. + """ + rows = _scope_rows(rt) + out = [] + for e in _kc().list_entries(rt): + scope, owner = entry_scope(rt, e["id"], rows) + if not owner: + owner = str(e.get("createdBy") or "") + if not may_see(scope, owner, uname): + continue + row = {**e, "scope": scope, "owner": owner} + if not (is_admin or str(owner) == str(uname)): + row["preview"] = "" + out.append(row) + return out + + +def _write_scope(rt, entry_id, scope, owner): + """Persist one entry's scope. A `business` entry CLEARS its row rather than writing the + default, so the store holds one spelling of the common case.""" + def _up(cur): + if scope == DEFAULT_SCOPE: + cur.pop(str(entry_id), None) + else: + cur[str(entry_id)] = {"scope": scope, "owner": str(owner or "")} + return cur + + rt.update(SCOPE_KEY, _up, flush="sync") + return True + + +def _may_touch(session, row): + """May this session change or delete `row`? An admin owns the business-wide ones; a member + owns their own personal ones. Anything else is not theirs to move.""" + if row.get("scope") == "personal": + return str(row.get("owner") or "") == str(session.uname) + return bool(session.admin) + + +def _kc(): + import core.keychain as keychain + return keychain + + +def _resolved_odoo_key(rt): + """`(source, flag_key)` — which source would serve this tenant's Odoo queries, in both the + shapes this module needs: the display string (`env` / `keychain:`) and the key the pause + flag is stored under. + + ⭐ D-10 (wave 24): THE RESOLUTION ITSELF NOW LIVES IN ONE PLACE, `TenantRuntime.odoo_flag_key`, + beside the `odoo_source()` it must agree with. This function had its own copy of the same + three rules — first unlocked keychain odoo entry, else env for tenant #0, else nothing — and + a second copy is exactly how a pause flag comes to be written against one resolution and read + against another, freezing nothing while the UI reports success. The two SHAPES stay here + because they are this module's presentation concern; the DECISION does not. + """ + flag_key = rt.odoo_flag_key() + if not flag_key: + return None, None + return ("env" if flag_key == _ENV_ODOO_FLAG_KEY else f"keychain:{flag_key}"), flag_key + + +def odoo_paused(rt): + """True when the tenant's RESOLVED Odoo source carries the pause flag. Pausing an entry + that is not the resolved source freezes nothing — it serves nothing. + + ⭐ D-10: a thin delegate now. The implementation moved to `TenantRuntime.odoo_paused` so the + measure mirror (`harness/datastore.py`, which cannot import this layer) asks the SAME question + the customer pool does. This name stays because `routes_customers`, `routes_products` and + `verify_api` all call it — moving the logic without moving the door keeps one answer and + costs no caller a change. + """ + return bool(rt.odoo_paused()) + + +def _snap_scope_key(team_id, agent): + return f"t={team_id}|a={agent}" + + +def load_pool_snapshot(rt, team_id, agent): + """(ts, rows) from the persisted snapshot for this exact scope, or None. NEVER a wider + scope's rows — serving the consolidated snapshot to a scoped user would widen their book.""" + try: + snap = (rt.get(SNAPSHOT_KEY) or {}).get("odoo_pool") or {} + e = snap.get(_snap_scope_key(team_id, agent)) + if isinstance(e, dict) and isinstance(e.get("rows"), list): + return float(e.get("ts") or 0), e["rows"] + except Exception: + pass + return None + + +def save_pool_snapshots(rt, taken_by=""): + """Persist every currently-cached pool scope as the pause-time snapshot ('the last + successful sync', made concrete). Ensures the consolidated default scope exists first so + a pause on a cold process still captures something to serve.""" + import time as _time + import routes_customers as _rc + try: + _rc._pool_for(rt, None, None) # the scope every admin/all-BU account lands on + except Exception: + pass # cold + Odoo down: persist whatever IS cached + pools = {} + for key, entry in list(rt.pool_cache.items()): + if (isinstance(key, tuple) and len(key) == 3 and key[0] == "pool" + and isinstance(entry, tuple) and len(entry) == 2 + and isinstance(entry[1], list)): + pools[_snap_scope_key(key[1], key[2])] = {"ts": entry[0], "rows": entry[1]} + if not pools: + return 0 + + def _up(cur): + cur["odoo_pool"] = pools + cur["taken"] = _time.strftime("%Y-%m-%dT%H:%M:%S") + cur["takenBy"] = str(taken_by or "") + return cur + + rt.update(SNAPSHOT_KEY, _up, flush="sync") + return len(pools) + + +def _rel_reconnect(rt): + """Lift the W32-T16 freeze. Its own function so `add_key` and the reconnect route cannot + disagree about what "resume" means.""" + import odoo_relational as rel + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + cur["frozen"] = False + cur.pop("frozenAt", None) + cur.pop("frozenBy", None) + return cur + + rt.update(rel.CONFIG_KEY, _up, flush="sync") + return True + + +# ══════════════════════════════════════════════════ W35-T45 / R11: THE ENV -> KEYCHAIN MIGRATION +# +# R11: *"Tenant #0's Odoo credential MIGRATES onto the keychain, with the environment kept as +# fallback."* The owner chose this over a read-only display row WITH THE MIGRATION RISK STATED, so +# the risk is what this block is mostly about. +# +# ⭐ WHAT WAS ALREADY TRUE, CHECKED BEFORE ANY OF IT WAS WRITTEN: the keychain-first-then-env +# RESOLVER has existed since the 2026-08-04 cutover. `harness/runtime.py::odoo_source` is already +# (1) a keychain `odoo` entry, (2) tenant #0's compiled env connector, (3) None for anybody else, +# and `visible_entries` already serves a non-admin the row WITHOUT a preview. So R11 is not "build +# a resolver" — it is "give tenant #0 the ROW", which is the only reason its Keychain page looks +# empty while its Odoo grids work. +# +# ⛔⛔ AND THE ONE REAL HAZARD IS NOT THE CREDENTIAL, IT IS THE PAUSE FLAG. `odoo_flag_key()` returns +# the first keychain `odoo` entry id when one exists and `ENV_ODOO_FLAG_KEY` ("odoo-env") otherwise — +# so CREATING THE ENTRY MOVES THE ADDRESS THE PAUSE FLAG LIVES AT. A tenant #0 that was paused under +# `odoo-env` would come back UNPAUSED, silently, at the first boot after this ships: the connector +# resumes pulling live Odoo because a migration changed which key the freeze was stored under. That +# is D-259's exact shape (a pause written under one key and read under another) and +# [[a-guard-bound-to-a-role-stops-guarding-when-the-role-moves]]. The flag is carried across in the +# SAME pass, and the carry is asserted. + + +def _env_odoo_fields(): + """The four env values `odoo_client` reads, or `(None, why)` when they are not all present. + + ⛔ ALL FOUR OR NOTHING, and this completeness check is load-bearing rather than defensive. + Creating the entry makes `odoo_source` resolve through branch 1 INSTEAD of the env — so a + PARTIAL migration would hand the connector `{url, db}` and no key and take tenant #0's Odoo + offline, on a deployment where it had been working. The env fallback cannot save it, because the + entry's existence is what turns the fallback off. + ⚠ The names are `odoo_client.py`'s own (`ODOO_URL`/`ODOO_DB`/`ODOO_USER`/`ODOO_API_KEY`) and the + field names are `harness/connectors/odoo.py`'s stored shape (`{url, db, user, api_key}`). Two + vocabularies meet here; nowhere else. + """ + want = (("url", "ODOO_URL"), ("db", "ODOO_DB"), + ("user", "ODOO_USER"), ("api_key", "ODOO_API_KEY")) + got = {field: (os.environ.get(env) or "").strip() for field, env in want} + missing = sorted(env for field, env in want if not got[field]) + if missing: + return None, (f"the environment is missing {', '.join(missing)}, and a partial credential " + f"would take this tenant's Odoo offline rather than migrate it") + return got, "" + + +def migrate_env_odoo(rt): + """R11 — put tenant #0's environment Odoo credential on its keychain, once. Returns a report. + + `{"done": bool, "entry": id|"", "carried_pause": bool, "why": str}` — `why` is filled on every + path including the skips, because "already migrated", "no keychain key on this deployment" and + "the env is incomplete" are three different operator actions. + + ⛔⛔ IT MUST RUN IN THE CONTAINER, WHICH IS WHY `main.py` CALLS IT AND NO SCRIPT DOES. D-195, + measured three times: a developer's CLI write to the tenant store is reverted by the running + Space within a minute (download-modify-upload, last-write-wins) — and **the write reports success + every time**, then a fresh read confirms it, and it is gone by the next poll. A CLI migration here + would be a dry run that lies, and the thing it would lie about is a credential. + + ⚠ FAIL-QUIET AND IDEMPOTENT. It runs on EVERY boot; the second one must be a no-op and a + third-party failure must not take the boot down. + """ + out = {"done": False, "entry": "", "carried_pause": False, "why": ""} + if not env_odoo_available(rt): + # Not tenant #0, or this deployment has no env Odoo at all. Both are normal states. + out["why"] = "this tenant has no environment Odoo credential to migrate" + return out + fields, why = _env_odoo_fields() + if not fields: + out["why"] = why + return out + # ⛔ READ THE PAUSE FLAG BEFORE THE WRITE. After the entry exists, `odoo_flag_key()` answers the + # NEW key and the old one is unreachable through the resolver — so the only moment this fact can + # be observed is now. [[undo-capture-before-the-write]] applied to a guard rather than to data. + try: + was_paused = bool(((rt.get(_CONNECTOR_FLAGS_KEY) or {}) + .get(_ENV_ODOO_FLAG_KEY) or {}).get("paused")) + except Exception: # noqa: BLE001 + was_paused = False + row, why = _kc().ensure_entry_of_type( + rt, "odoo", "Odoo (migrated from this deployment)", fields, "system") + if row is None: + out["why"] = why + return out + out["done"], out["entry"] = True, row["id"] + if was_paused: + # The freeze followed the credential. Without this the connector silently RESUMES pulling + # live Odoo at the first boot after the migration. + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + entry = dict(cur.get(row["id"]) or {}) + entry["paused"] = True + entry["pausedBy"] = "system" + entry["pausedNote"] = ("carried over from the environment source when the credential " + "was migrated onto the keychain") + cur[row["id"]] = entry + return cur + + try: + rt.update(_CONNECTOR_FLAGS_KEY, _up, flush="sync") + out["carried_pause"] = True + except Exception as exc: # noqa: BLE001 + # ⛔ SAID OUT LOUD. A migration that moved the credential and lost the freeze is worse + # than one that did not run, so this is the one failure that must never be silent. + out["why"] = (f"the credential migrated but the PAUSE could not be carried over " + f"({type(exc).__name__}), so this tenant's Odoo is no longer frozen") + return out + + +def _own_row(session, entry_id): + """The visible row for `entry_id`, or a 404. ⛔ A 404 rather than a 403 for an entry the + caller cannot see: telling a member that somebody else's personal credential EXISTS is the + disclosure the scope is for.""" + row = next((e for e in visible_entries(session.runtime, session.uname, + bool(session.admin)) + if e["id"] == str(entry_id)), None) + if row is None: + raise err(404, "no_entry", "no such key") + return row + + +@router.get("/admin/keychain") +def list_keychain(session: Session = Depends(require_session)): + """⭐ W32-T11 / R4 — SESSION-GATED, NOT ADMIN-GATED, and that is the ruling not a relaxation. + R4 puts a PERSONAL connection in every member's hands, so a room only an admin can open would + ship the feature and no door to it. The wall moved INTO the payload: a member sees the + business-wide entries and their own, never anybody else's personal one.""" + kc = _kc() + return {"entries": visible_entries(session.runtime, session.uname, + bool(session.admin)), + "locked": not kc.unlocked(), + #: the vocabulary and the permission, so the client renders a picker it can honour + #: rather than offering an option the server will refuse (R4: business is admin-only). + "scopes": list(SCOPES), "canBusiness": bool(session.admin), + "tenantWideTypes": list(TENANT_WIDE_TYPES)} + + +@router.post("/admin/keychain", status_code=201) +def add_key(body: dict = Body(default=None), session: Session = Depends(require_session)): + kc = _kc() + body = body or {} + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable") + scope = clean_scope(body.get("scope")) + if scope is None: + raise err(400, "bad_scope", + f"scope must be one of {', '.join(SCOPES)}") + if scope == "business" and not session.admin: + raise err(403, "not_admin", + "a business-wide connection applies to everyone in this workspace, so only an " + "administrator can create one. You can add it as a personal connection instead.") + etype = str(body.get("type") or "").strip().lower() + if scope == "personal" and etype in TENANT_WIDE_TYPES: + # ⛔ REPORTED, NOT COERCED (W30/R6's second sentence). See TENANT_WIDE_TYPES above: this + # credential is what the WHOLE workspace's databases are built from, so "personal" would + # be a label on a tenant-wide key rather than a boundary around it. + raise err(400, "scope_not_available", + f"a {etype} connection is what this whole workspace's databases are read " + f"through, so it is always business-wide — it cannot be a personal connection. " + f"An administrator can add it for everyone.") + try: + row = kc.add_entry(session.runtime, body.get("label"), body.get("type"), + body.get("fields"), session.uname) + except kc.KeychainLocked as e: + raise err(503, "keychain_locked", + f"the keychain is locked — {e}. A secret is never stored unencrypted.") + except ValueError as e: + raise err(400, "bad_entry", str(e)) + except Exception: + raise err(503, "store_unavailable", "the entry was not saved — try again") + # ⛔⛔ A PERSONAL ENTRY THAT LOSES ITS SCOPE ROW READS AS BUSINESS-WIDE — i.e. the failure mode + # of a side bucket is to publish a credential, not to hide one. So the second write is not + # best-effort: if it does not land, the entry is REMOVED and the caller is told nothing was + # stored. `business` needs no row at all, so this branch is the only one that can be partial. + # ⭐ W32-T16 / R10's SECOND SENTENCE — *"Reconnecting resumes into the same tables."* Storing + # an Odoo credential IS reconnecting, so it lifts the freeze here rather than making the admin + # find a second switch. The tables were never dropped, so "resume" is one flag. + if etype == "odoo": + try: + _rel_reconnect(session.runtime) + except Exception: # noqa: BLE001 + pass + if scope != DEFAULT_SCOPE: + try: + _write_scope(session.runtime, row["id"], scope, session.uname) + except Exception: + try: + kc.delete_entry(session.runtime, row["id"]) + except Exception: # noqa: BLE001 + pass + raise err(503, "store_unavailable", + "the key was not saved — its sharing setting could not be stored, so " + "nothing was kept. Try again.") + return {"entry": {**row, "scope": scope, "owner": session.uname}} + + +@router.put("/admin/keychain/{entry_id}") +def update_key(entry_id: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Contract C1's scope door. Only the scope moves — a stored secret is never re-openable, so + "edit this key" means "replace it" and that is `DELETE` + `POST`.""" + row = _own_row(session, entry_id) + scope = clean_scope((body or {}).get("scope"), default=None) + if scope is None: + raise err(400, "bad_scope", f"scope must be one of {', '.join(SCOPES)}") + if scope == "business" and not session.admin: + raise err(403, "not_admin", + "a business-wide connection applies to everyone in this workspace, so only an " + "administrator can make one business-wide.") + if not _may_touch(session, row): + raise err(403, "not_yours", "this connection is not yours to change") + if scope == "personal" and str(row.get("type") or "") in TENANT_WIDE_TYPES: + raise err(400, "scope_not_available", + f"a {row.get('type')} connection is what this whole workspace's databases are " + f"read through, so it is always business-wide.") + owner = row.get("owner") or session.uname + try: + _write_scope(session.runtime, entry_id, scope, owner) + except Exception: + raise err(503, "store_unavailable", "the change was not saved — try again") + return {"entry": {**row, "scope": scope, "owner": owner if scope == "personal" else ""}} + + +@router.delete("/admin/keychain/{entry_id}") +def delete_key(entry_id: str, session: Session = Depends(require_session)): + row = _own_row(session, entry_id) + if not _may_touch(session, row): + raise err(403, "not_yours", "this connection is not yours to delete") + try: + _kc().delete_entry(session.runtime, entry_id) + _write_scope(session.runtime, entry_id, DEFAULT_SCOPE, "") # drop the side row with it + except Exception: + raise err(503, "store_unavailable", "the delete did not land — try again") + return {"ok": True} + + +@router.post("/admin/keychain/{entry_id}/test") +def test_key(entry_id: str, session: Session = Depends(require_session)): + _own_row(session, entry_id) # 404 for an entry this caller may not see + return _kc().test_entry(session.runtime, entry_id) + + +def _unsynced_customer_records(session): + """R3's guardrail, tenant #0's customer topic: overlay-holding pids the CURRENT pool no + longer serves. Overlays are unioned across EVERY user of the table (the guardrail is a + tenant fact, not a per-user one). Honest degradation: when the pool cannot be built the + answer is `known: False`, never a fabricated zero.""" + try: + import core.table_store as table_store + bucket = session.runtime.get("customer_table_workspace") or {} + overlay_pids = {} + for uname, ws in bucket.items(): + if uname == table_store.SHARED_KEY or not isinstance(ws, dict): + continue + for pid, cells in (ws.get("overlays") or {}).items(): + if isinstance(cells, dict) and cells: + overlay_pids.setdefault(str(pid), cells) + if not overlay_pids: + return {"known": True, "count": 0, "rows": []} + from routes_customers import allowed_pids + pool = {str(p) for p in allowed_pids(session)} + orphans = sorted((p for p in overlay_pids if p not in pool), key=lambda x: int(x) + if str(x).isdigit() else 0) + rows = [] + for p in orphans[:50]: + cells = overlay_pids[p] + hint = next((str(v) for v in cells.values() if str(v).strip()), "") + rows.append({"pid": int(p) if str(p).isdigit() else p, + "fields": len(cells), "hint": hint[:80]}) + return {"known": True, "count": len(orphans), "rows": rows, + "shown": min(len(orphans), 50)} + except Exception as e: + return {"known": False, "count": None, "rows": [], + "note": f"pool unavailable — {type(e).__name__}"} + + +@router.get("/admin/connectors") +def connectors(session: Session = Depends(require_session)): + kc = _kc() + flags = session.runtime.get(_CONNECTOR_FLAGS_KEY) or {} + # ⭐ W32-T11 / R4 — THE SAME VISIBILITY RULE AS THE KEYCHAIN, because this pane is the same + # facts with a status column. Reading `kc.list_entries` here instead would have shown a member + # every colleague's personal connection on the screen next door to the one that hides them. + entries = visible_entries(session.runtime, session.uname, bool(session.admin)) + # R3 cutover (2026-08-04): which source would actually serve this tenant's Odoo queries — + # mirrors TenantRuntime.odoo_source() exactly: first unlocked keychain odoo entry, else env + # for tenant #0 only, else nothing (fail closed — never another tenant's environment). + # ⚠ W32-T11: computed over the TENANT's entries, not over `entries` above. "Which source + # serves this workspace" is one fact for everybody, and deriving it from a per-USER list would + # make the answer depend on who opened the pane. Personal entries are excluded for the same + # reason `TENANT_WIDE_TYPES` refuses them: they must never become the workspace's source. + _scopes = _scope_rows(session.runtime) + first_odoo = next((e["id"] for e in kc.list_entries(session.runtime) + if e["type"] == "odoo" + and entry_scope(session.runtime, e["id"], _scopes)[0] != "personal"), None) + # ⛔ W32-T10 — the env leg goes through `env_odoo_available` now, so this route and the + # connectors DIRECTORY answer "does the environment serve this tenant?" with one function + # instead of two spellings that agreed until a second tenant signed in. + if first_odoo and kc.unlocked(): + resolved = f"keychain:{first_odoo}" + elif env_odoo_available(session.runtime): + resolved = "env" + else: + resolved = None + rows = [] + if env_odoo_available(session.runtime): + rows.append({"key": _ENV_ODOO_FLAG_KEY, "label": "Odoo (environment)", "type": "odoo", + "source": "env", "active": resolved == "env", + # the deployment's own credential — business-wide by construction, and it + # has no owner to be personal to. + "scope": DEFAULT_SCOPE, "owner": "", + "paused": bool((flags.get(_ENV_ODOO_FLAG_KEY) or {}).get("paused"))}) + for e in entries: + rows.append({"key": e["id"], "label": e["label"], "type": e["type"], + "source": "keychain", "preview": e["preview"], + "scope": e.get("scope") or DEFAULT_SCOPE, "owner": e.get("owner") or "", + "active": (e["type"] == "odoo" and resolved == f"keychain:{e['id']}"), + "paused": bool((flags.get(e["id"]) or {}).get("paused"))}) + out = {"connectors": rows, "locked": not kc.unlocked(), "resolved": resolved, + "scopes": list(SCOPES), "canBusiness": bool(session.admin), + # ⭐ D-10 (wave 24) — THIS SENTENCE IS NOW TRUE OF EVERY PATH, which it was not before. + # DEBT-2 (2026-08-04) froze the CUSTOMER pool and this note honestly disclosed the + # hole it left: "measures not already computed may still reach the source". D-10 + # closed that hole — `harness/datastore.py` (the mirror every measure column is + # answered from) refuses to sync while paused, and `routes_products._pool_for` got the + # guard its customer sibling has had since DEBT-2. So the caveat is deleted rather + # than left standing, because a warning that outlives its defect teaches the reader to + # ignore warnings. + # ⚠ THE THREE BEHAVIOURS ARE NAMED SEPARATELY on purpose: they are genuinely + # different answers (a persisted snapshot, an in-process cache, a frozen mirror), and + # collapsing them into "everything freezes" would be the kind of tidy summary that + # stops being true the first time one of them changes. + # ⭐ D-62 CLOSED (wave 27) — AND THE REGISTER'S DIAGNOSIS OF IT WAS WRONG, so the + # correction is recorded here rather than silently applied. D-62 said this note + # "promises a behaviour on a dashboard measure path that has been dead since W16". + # MEASURED 2026-08-08, and it is not: measure COLUMNS are grid columns answered from + # the DuckDB mirror, and `harness/datastore.py` genuinely refuses to sync while + # paused (`source_paused()` at four sites), so that clause was TRUE. The dead path is + # `/api/v1/pages/{key}` (D-52), which this note never mentioned. + # + # ⛔ THE REAL DEFECT WAS THE OPPOSITE ONE, and it was the last sentence: "so figures + # stop moving rather than going blank". BOTH pool paths answer **503** when they have + # no copy to serve — the customer path for a scope with no snapshot + # (`routes_customers.py:88`) and the product path ALWAYS after a restart, because + # there is no product snapshot bucket at all (`routes_products.py:60-73`, which says + # so in as many words). So a paused connector plus a restarted server is exactly the + # blank screen this sentence promised could not happen. A warning that over-promises + # is worse than none: it is the sentence somebody quotes when the screen disagrees. + "pausedNote": ("Pausing a connector never deletes data — notes, custom fields and " + "views stay, and nothing reaches the source while it is paused. " + "Anything this server has already read keeps showing: the customer " + "workspace serves its pause-time snapshot, the product list serves " + "the last copy read since startup, and measure columns keep answering " + "from the mirror as it stood when you paused. What has NOT been read " + "cannot be shown — a scope with no snapshot, or the product list after " + "a restart, reports that the source is paused instead of showing " + "figures. Resume to start reading live again.")} + # ⛔⛔ W32-T11 — ADMIN-GATED, AND THIS IS A DISCLOSURE FIX, NOT TIDINESS. Opening this route to + # members (R4) opened this block with it, and `_unsynced_customer_records` is the one thing on + # the payload that is NOT about connectors: it unions overlays across EVERY user of the + # customer table — its own docstring says so, *"the guardrail is a tenant fact, not a per-user + # one"* — and returns `hint`, the first non-empty cell of somebody else's overlay. So a member + # would have read colleagues' typed notes off the Connectors pane. It also filters against + # `allowed_pids(session)`, so a BU-scoped member's narrower pool inflates the orphan count and + # the number itself becomes wrong for them as well as private. + # ⚠ The lesson generalises past this line: opening a route widens EVERY field it already + # returned, and the audit has to walk the payload, not the entry list I was thinking about. + if session.tenant == "royal-imports" and session.admin: + out["unsynced"] = _unsynced_customer_records(session) + return out + + +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ W32-T15/T16/T17 / CONTRACT C2 / RULINGS R9, R10, R11 — THE ODOO CONNECTOR ACTUALLY OPENS +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# +# The owner clicked "Manage keys" on Odoo and found a credential list. Not: which server database +# this workspace reads, which of the ten mirrored grids it wants, how often they sync, or how to +# stop. Every decision below lives in `odoo_relational` (the module `refresh()` reads) so that a +# switch flipped here is a switch the sync path obeys — a config the route knows and the sync +# path does not is a control that does nothing and reports success. +def _rel(): + import odoo_relational as rel + return rel + + +def _odoo_entry(session): + """The keychain entry SERVING this tenant's Odoo, or None when the environment is (or nothing + is). Business-scoped by construction — `TENANT_WIDE_TYPES` refuses a personal one.""" + kc = _kc() + scopes = _scope_rows(session.runtime) + for e in kc.list_entries(session.runtime): + if e["type"] == "odoo" and entry_scope(session.runtime, e["id"], scopes)[0] != "personal": + return e + return None + + +def _odoo_source_fields(session, entry): + """`(serverDb, serverUrl, apiUser, editable)` — what the panel may SHOW about the connection. + + ⛔ NEVER THE SECRET. `read_fields` is documented as the connector layer's internal and no + route returns its output; this returns the three fields that identify WHICH server, and the + api key is not among them. The masked preview is the entry's own and was computed at write. + ⚠ The ENVIRONMENT source is not editable and says so: it is the deployment's `.env`, shared by + the process, and an admin editing it from a tenant screen would be editing the container. + """ + if entry is None: + return (os.environ.get("ODOO_DB", ""), os.environ.get("ODOO_URL", ""), + os.environ.get("ODOO_USER", ""), False) + try: + f = _kc().read_fields(session.runtime, entry["id"]) or {} + except Exception: # noqa: BLE001 + return ("", "", "", True) # locked keychain: honest blanks, still editable + return (str(f.get("db") or ""), str(f.get("url") or ""), str(f.get("user") or ""), True) + + +def _odoo_admin(session): + """C2's doors are admin doors: they show the credential that serves EVERYONE and can turn the + whole workspace's databases off. R4's personal scope has nothing to say here — a tenant-wide + type cannot be personal in the first place.""" + if not session.admin: + raise err(403, "not_admin", + "the Odoo connection serves this whole workspace, so only an administrator can " + "configure it") + + +@router.get("/admin/connectors/odoo/config") +def odoo_config(session: Session = Depends(require_session)): + """Contract C2's read: `{serverDb, grids, syncEvery, canDisconnect}` and the rest of what a + person needs to see before changing any of it.""" + _odoo_admin(session) + rel = _rel() + entry = _odoo_entry(session) + server_db, server_url, api_user, editable = _odoo_source_fields(session, entry) + cfg = rel.read_config(session.runtime) + return { + "applicable": bool(rel.is_royal(session.tenant)), + "source": "keychain" if entry else ("env" if env_odoo_available(session.runtime) + else "none"), + "entryId": (entry or {}).get("id", ""), + "label": (entry or {}).get("label", "Odoo (environment)"), + "preview": (entry or {}).get("preview", ""), + "serverDb": server_db, "serverUrl": server_url, "apiUser": api_user, + "serverDbEditable": editable, + "grids": rel.grid_choices(session.runtime), + "syncEvery": cfg["syncEvery"], + "syncOptions": list(rel.SYNC_PRESETS), + "syncFloorSeconds": rel.SYNC_FLOOR_SECONDS, + "frozen": cfg["frozen"], "frozenAt": cfg["frozenAt"], + # ⚠ There is nothing to disconnect FROM when the source is the deployment environment: + # tenant #0's `.env` is not this tenant's to remove. Said as a field so the client renders + # no button rather than one that 400s. + "canDisconnect": bool(entry) or (env_odoo_available(session.runtime) + and not cfg["frozen"]), + } + + +@router.put("/admin/connectors/odoo/config") +def odoo_config_put(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Contract C2's write. Grids, cadence and the server database — each optional, each REPORTED + back rather than silently applied.""" + _odoo_admin(session) + rel = _rel() + body = body or {} + notes = [] + + grids = body.get("grids") + known = {c["key"] for c in rel.grid_choices(session.runtime)} + clean_grids = None + if isinstance(grids, dict): + unknown = sorted(str(k) for k in grids if str(k) not in known) + if unknown: + # ⛔ NAMED, NOT DROPPED. A key we do not serve is a client that believes in a grid + # this connector does not have, and swallowing it makes the two disagree quietly. + raise err(400, "unknown_grid", + f"this connector has no grid called {', '.join(unknown)}") + clean_grids = {str(k): bool(v) for k, v in grids.items()} + if clean_grids and not any(clean_grids.get(k, True) for k in known): + notes.append("every grid is switched off — nothing will be materialised on the next " + "sync, and the databases you already have are left untouched") + + every = body.get("syncEvery") + clean_every = None + if every is not None: + clean_every = str(every).strip().lower() + if clean_every not in rel.SYNC_PRESETS: + # ⛔⛔ R11 + W30/R6's SECOND SENTENCE: the floor is enforced AND the caller is told. + # A crafted `"5m"` is CLAMPED to the floor and the response says so — never applied, + # and never silently ignored either, because a control that discards your answer + # without a word is how a limit becomes invisible. + clean_every = rel.DEFAULT_SYNC + notes.append(f"{every!r} is not an interval this connector offers, and anything under " + f"{rel.SYNC_FLOOR_SECONDS // 60} minutes is not available at all — the " + f"sync interval was set to the {rel.DEFAULT_SYNC} floor instead") + + server_db = body.get("serverDb") + if server_db is not None: + server_db = " ".join(str(server_db).split())[:80] + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + if clean_grids is not None: + cur.setdefault("grids", {}).update(clean_grids) + if clean_every is not None: + cur["syncEvery"] = clean_every + return cur + + try: + session.runtime.update(rel.CONFIG_KEY, _up, flush="sync") + except Exception: + raise err(503, "store_unavailable", "the change was not saved — try again") + + if server_db: + notes.append(_rewrite_server_db(session, server_db)) + + # ⭐⭐ W33-T65 / W30-R6's SECOND SENTENCE — THE CADENCE IS SET AND ONLY PARTLY OBEYED, AND THE + # PERSON SETTING IT IS THE ONE WHO HAS TO BE TOLD. Measured, not guessed: + # · `main.py::_store_resync_loop` reads the interval from `sync_seconds(get_runtime( + # "royal-imports"))` — a HARDCODED slug — and then sleeps ONCE for the whole process. So + # for tenant #0 this control moves EVERYBODY's sync, and for every other tenant the value + # is stored, clamped, displayed and never read. + # · `manual` stores and reads back as `None`, and the loop has no branch on it: it sleeps a + # default 1800 s and syncs anyway. "Only when I ask" asks all the same. + # ⛔ NEITHER IS FIXABLE FROM THIS FILE — the loop lives in `main.py`, which this lane does not + # own — and shipping a setting that silently does nothing is the exact failure R6 names. So it + # is REPORTED here, at the moment of the change, with what it really controls. Delete these + # notes when the loop becomes per-tenant, not before. + if clean_every is not None: + if not rel.is_royal(session.tenant): + notes.append("this interval is saved, but the sync loop currently reads its schedule " + "from one workspace for the whole deployment — so it will not change how " + "often YOUR data refreshes until per-workspace scheduling ships") + else: + notes.append("this interval is saved and it is the one the deployment's sync loop " + "uses — it changes the refresh rate for every workspace on this " + "deployment, not only this one") + if clean_every == "manual": + notes.append("⚠ 'manual' does not yet stop the background sync: the loop has no " + "manual-only branch, so data still refreshes on the default interval") + + out = odoo_config(session) + return {**out, "notes": [n for n in notes if n]} + + +def _rewrite_server_db(session, server_db): + """Point the stored Odoo credential at a different server database (R9's first reading). + + ⛔ THERE IS NO "UPDATE ENTRY" IN THE KEYCHAIN, and writing one here would be a SECOND copy of + how a secret is encrypted and previewed — the thing `core/keychain.py` exists to hold alone. + So this is add-then-delete through the module's own doors, with the side rows (pause flag, + scope) carried across because they are keyed by ENTRY ID. + ⚠ THE ORDER IS DELIBERATE AND THE WINDOW IS REAL: for the moment between the add and the + delete this tenant has TWO odoo entries, and `odoo_creds` takes the first by id sort — so a + resync landing inside that window could read the OLD database. The alternative order can + leave the workspace with no credential at all, which is worse than one stale read. Milliseconds + of ambiguity beats a lost key. + """ + kc = _kc() + entry = _odoo_entry(session) + if entry is None: + return ("the server database is set on this deployment's environment, not in the " + "keychain, so it was not changed here") + try: + fields = kc.read_fields(session.runtime, entry["id"]) or {} + except kc.KeychainLocked as e: + raise err(503, "keychain_locked", f"the keychain is locked — {e}") + if not fields: + raise err(400, "bad_entry", "this credential could not be read back to be changed") + if str(fields.get("db") or "") == server_db: + return "" + fields["db"] = server_db + try: + new = kc.add_entry(session.runtime, entry["label"], "odoo", fields, session.uname) + except Exception: + raise err(503, "store_unavailable", + "the server database was not changed — the existing connection is untouched") + # carry the side rows across, then retire the old entry + try: + flags = session.runtime.get(_CONNECTOR_FLAGS_KEY) or {} + if entry["id"] in flags: + def _mv(cur): + cur[new["id"]] = cur.pop(entry["id"], {}) + return cur + session.runtime.update(_CONNECTOR_FLAGS_KEY, _mv) + kc.delete_entry(session.runtime, entry["id"]) + _write_scope(session.runtime, entry["id"], DEFAULT_SCOPE, "") + except Exception: # noqa: BLE001 + return (f"the connection now points at {server_db}, but the previous credential could " + f"not be removed — delete it under Keychains") + return f"the connection now points at the {server_db} database" + + +@router.post("/admin/connectors/odoo/disconnect") +def odoo_disconnect(session: Session = Depends(require_session)): + """R10 — remove the credential and FREEZE the grids as static data. + + ⛔ DISTINCT FROM PAUSE, and the difference is the credential. Pause is temporary and keeps the + key; disconnect deletes it and marks the databases frozen so nothing refreshes them again — + including the boot rebuild and the resync loop, which for tenant #0 would otherwise + re-materialise from the process ENVIRONMENT and quietly undo the disconnect. + ⛔⛔ AND IT DELETES NOTHING ELSE. The owner's words are *"so we don't fuck up"*: every row and + every FIELD DEFINITION stays, user-added columns included, because a field a person added is + the thing a naive freeze drops first. This route never touches `fields` or `rows` — it writes + one flag in a different bucket, which is what makes that guarantee structural rather than + careful. + """ + _odoo_admin(session) + rel = _rel() + import datetime as _dt + entry = _odoo_entry(session) + if not entry and not env_odoo_available(session.runtime): + raise err(400, "not_connected", "this workspace has no Odoo connection to disconnect") + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + cur["frozen"] = True + cur["frozenAt"] = _dt.datetime.now().strftime("%Y-%m-%dT%H:%M:%S") + cur["frozenBy"] = str(session.uname) + return cur + + # ⭐⭐ W33-T65 — THE FREEZE HAD A HOLE, AND THE NOTE BELOW WAS THE THING THAT MADE IT A DEFECT + # RATHER THAN A LIMIT. `rel.frozen` has exactly ONE consumer, `odoo_relational.refresh`, which + # materialises the EIGHT copied grids. The other two (`ut_odoo_order_lines`, + # `ut_odoo_gl_lines` — `READ_THROUGH_KEYS`) do not go through `refresh` at all: they read + # THROUGH the per-tenant DuckDB mirror, and the mirror is advanced by `datastore.sync_all`, + # which gates on the connector PAUSE flag and has never heard of `frozen`. So a disconnected + # workspace kept serving LIVE, still-moving rows in its two biggest grids while this route's + # own sentence promised *"nothing is being refreshed"*. + # + # ⛔ THE FIX IS TO MAKE THE SENTENCE TRUE, not to soften it. Disconnect now flips the pause + # flag on the RESOLVED source as well, which is the switch `sync_all` and `reconcile_deletes` + # actually read — so both halves of "frozen" mean the same thing. Two orderings are borrowed + # from `pause_connector` because it learned them the hard way: + # · the flag key is resolved BEFORE the credential is deleted — after the delete there is no + # resolved source left to name, and the flag would land under a key nothing reads (D-10). + # · the snapshot is captured BEFORE the flag flips, so there is a last-successful-sync to + # serve; a failed capture leaves the connector live rather than paused-with-nothing. + _, flag_key = _resolved_odoo_key(session.runtime) + snapshots = 0 + if flag_key: + try: + snapshots = save_pool_snapshots(session.runtime, taken_by=session.uname) + except Exception: # noqa: BLE001 + # A snapshot is a nicety; the freeze is the promise. Reported, never fatal. + snapshots = 0 + + # ⚠ THE FLAG FIRST, THE CREDENTIAL SECOND. If the flag write fails, nothing has happened and + # the connector is still live; if the delete failed AFTER the flag landed, the tenant is + # frozen with an unused key, which is recoverable from the Keychains pane. The reverse order + # can leave a workspace with no key and a connector that still tries to sync. + try: + session.runtime.update(rel.CONFIG_KEY, _up, flush="sync") + except Exception: + raise err(503, "store_unavailable", "nothing was disconnected — try again") + + paused_mirror = False + if flag_key: + def _pause(cur): + cur = cur if isinstance(cur, dict) else {} + cur[str(flag_key)] = {"paused": True} + return cur + try: + session.runtime.update(_CONNECTOR_FLAGS_KEY, _pause, flush="sync") + paused_mirror = True + except Exception: # noqa: BLE001 + paused_mirror = False + + removed = "" + if entry is not None: + try: + _kc().delete_entry(session.runtime, entry["id"]) + _write_scope(session.runtime, entry["id"], DEFAULT_SCOPE, "") + removed = entry["id"] + except Exception: + raise err(503, "store_unavailable", + "the databases are frozen but the stored credential was not removed — " + "delete it under Keychains") + + # ⛔ THE SENTENCE IS COMPOSED, NOT CONSTANT, because the two sources genuinely differ and the + # old fixed string was wrong about one of them. A tenant whose Odoo came from the DEPLOYMENT + # ENVIRONMENT has no credential for this route to remove — `.env` is the container's, not a + # tenant screen's — so it said "the key back" about a key it never held. R6's second sentence: + # the limit that cannot be removed is REPORTED, with what to do instead. + note = ("Your Odoo databases are frozen: every row and every column you had is still there and " + "still readable, and nothing is being refreshed.") + if not paused_mirror and flag_key: + note += (" ⚠ The live mirror could not be paused, so the two read-through databases " + "(order lines and GL lines) may keep advancing — pause the Odoo connector under " + "Keychains to stop them.") + note += (" Reconnecting adds the key back and resumes into the same databases." if entry + else " This workspace's Odoo credential comes from the deployment environment, so " + "there was no stored key to remove — the databases are frozen and Reconnect " + "resumes them into the same tables.") + return {"frozen": True, "removedEntry": removed, + # ⚠ ON THE WIRE, so the client and a gate can both see which half happened. A boolean + # nobody returns is a guarantee nobody can check. + "pausedMirror": paused_mirror, "snapshots": snapshots, + "source": "keychain" if entry else "env", + "note": note} + + +@router.post("/admin/connectors/odoo/reconnect") +def odoo_reconnect(session: Session = Depends(require_session)): + """R10's second sentence — *"Reconnecting resumes into the same tables."* + + It clears the freeze and nothing else: the tables were never dropped, so there is nothing to + recreate. A tenant whose source was a keychain entry adds it back under Keychains first; this + is the switch that lets the sync path see it again. + """ + _odoo_admin(session) + try: + _rel_reconnect(session.runtime) + except Exception: + raise err(503, "store_unavailable", "the change was not saved — try again") + # ⭐⭐ W33-T65 — AND THE PAUSE DISCONNECT SET, or the freeze would be one-way. Clearing only + # `frozen` restores the eight materialised grids and leaves the mirror pinned forever, so the + # two read-through grids would sit at the disconnect date while the panel said "connected + # again" — the same disagreement between the halves of "frozen", pointing the other way. + # ⚠ Resolved AFTER `_rel_reconnect`: a tenant reconnects by adding the key back FIRST, so the + # resolved source only exists again by this point. + _, flag_key = _resolved_odoo_key(session.runtime) + resumed = False + if flag_key: + def _unpause(cur): + cur = cur if isinstance(cur, dict) else {} + cur[str(flag_key)] = {"paused": False} + return cur + try: + session.runtime.update(_CONNECTOR_FLAGS_KEY, _unpause, flush="sync") + resumed = True + except Exception: # noqa: BLE001 + resumed = False + connected = bool(_odoo_entry(session)) or env_odoo_available(session.runtime) + return {"frozen": False, "connected": connected, "resumedMirror": resumed, + "note": ("Odoo is connected again and the databases you already had will refresh in " + "place." if connected else + "The freeze is lifted, but there is no Odoo credential yet — add one under " + "Keychains and the databases resume into the same tables.")} + + +@router.post("/admin/connectors/{key}/pause") +def pause_connector(key: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + paused = bool((body or {}).get("paused")) + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable") + # ⭐ W32-T11 / R4 — pausing a BUSINESS-WIDE connector stops it for everyone, so it stays an + # admin act; pausing your own personal one is yours. The env source has no keychain row and + # is business-wide by construction, hence the admin fallthrough. + row = next((e for e in visible_entries(session.runtime, session.uname, + bool(session.admin)) + if e["id"] == str(key)), None) + if row is not None: + if not _may_touch(session, row): + raise err(403, "not_yours", "this connection is not yours to pause") + elif not session.admin: + raise err(403, "forbidden", "administrators only") + + # DEBT-2: pausing the RESOLVED Odoo source captures the snapshot FIRST, so there is a + # "last successful sync" to serve before the freeze takes effect. Capturing before the + # flag flips means a failed capture leaves the connector live (never paused-with-nothing). + snapshots = 0 + _, flag_key = _resolved_odoo_key(session.runtime) + if paused and flag_key and str(key) == flag_key: + snapshots = save_pool_snapshots(session.runtime, taken_by=session.uname) + + def _up(cur): + cur[str(key)] = {"paused": paused} + return cur + + try: + session.runtime.update(_CONNECTOR_FLAGS_KEY, _up) + except Exception: + raise err(503, "store_unavailable", "the change was not saved — try again") + return {"key": key, "paused": paused, "snapshots": snapshots} diff --git a/api/routes_nav.py b/api/routes_nav.py index 8c0063d7ae24a6e239dd17fd6ee76e74d72f4926..25a055836ca0c8308102829e514ca499e2f079e1 100644 --- a/api/routes_nav.py +++ b/api/routes_nav.py @@ -1,989 +1,989 @@ -"""routes_nav.py — X2's `GET /api/v1/nav`: the registry, filtered by what this session may open. - -SERVER-FILTERED, not client-filtered. The client renders what it is given and never decides who -may see what — a nav that hides a link the API would still serve is a UI courtesy, not a -permission. `core.perms.nav_pages` is the single predicate (shared with `may_open`'s page gate), -so the nav and the 403 can never disagree about a grant. - -The full rule set — archived is invisible to everyone, `group_only` rows are excluded so no -`parent` reference can dangle, `nav: False` surfaces ship as `chrome: 'utility'`, and the -`prefs.json` Library preference is deliberately NOT applied — is documented on -`core.perms.nav_pages`, which is where it belongs: one place, both callers. -""" -import time - -from fastapi import APIRouter, Body, Depends - -from deps import Session, err, perms, require_session - -router = APIRouter(prefix="/api/v1") - -#: Wave 2026-08-02 (C-SCHEMA): per-user folders over the database list, the view-folder -#: pattern applied to the nav. Cosmetic per-user state — placement never grants or hides a -#: surface (the server-filtered nav still decides what exists). -_NAV_PREFS_KEY = "nav_prefs" -_MAX_NAV_FOLDERS = 16 - -#: WAVE 19 (R8 / contract C1): the database's NAME and ICON overrides. -#: -#: ⚠ TENANT-WIDE, which is the whole difference from `nav_prefs` above and the reason it is a -#: separate bucket rather than another field in that one. Folders and placement are one -#: person's arrangement of their own rail — per-user by definition. What a database is CALLED -#: and what it looks like are facts about the database: a workspace where two people call the -#: same table different things has no shared vocabulary left to discuss it in. Same store, two -#: buckets, because they answer to two different owners. -#: -#: Shape: {"": {"icon": {"shape": , "tone": }, -#: "name": ""}} -_NAV_META_KEY = "nav_meta" - -#: The grid's folder-icon vocabulary, mirrored — 12 shapes x 5 tones. The CLIENT imports these -#: from `customer-grid/types` rather than redefining them; this end cannot import TypeScript, -#: so it is the one place the list is written twice. -#: -#: ⛔ WHAT AN UNKNOWN VALUE MUST NOT DO IS BE STORED. `FolderMark` indexes its path table by -#: shape and maps the result, so a shape this whitelist let through and the renderer does not -#: know is `undefined.map()` — a blank rail, from a stored preference, for every user in the -#: tenant until somebody edits the store by hand. Refusing at the door is the cheap end of -#: that. If the two lists ever drift the symptom is an icon that silently reverts to the -#: default, which is the loudest SAFE failure available here. -_ICON_SHAPES = frozenset({"folder", "star", "flag", "tag", "bookmark", "grid", - "chart", "map", "users", "clock", "heart", "bolt"}) -_ICON_TONES = frozenset({"neutral", "blue", "green", "yellow", "red"}) -_MAX_NAV_NAME = 60 - -#: WAVE 23 (contract C10 / ruling R7) — the Home landing's RECENTS. -#: -#: PER-USER, like `nav_prefs` two buckets up and unlike `nav_meta`: what I opened last is -#: nobody else's business, and a tenant-wide "recently opened" would be a surveillance feature -#: rather than a convenience one. -#: -#: ⛔ A MAP KEYED BY PAGE, NOT AN APPEND LOG, and the difference is the whole feature. -#: `{username: {pageKey: }}` — re-opening a database OVERWRITES its stamp. An -#: append-only list capped at 50 fills with fifty copies of the same ten databases inside one -#: working session, and the cap then evicts OLDEST-FIRST: the tenth database you touched falls -#: off the list while forty slots hold repeat visits to the first. Keying by page makes -#: "recent" mean what the word means, and makes the cap bound the number of DATABASES -#: remembered rather than the number of clicks. -_NAV_RECENTS_KEY = "nav_recents" -_MAX_RECENTS = 50 - -#: ⚠ EPOCH SECONDS (UTC by definition), never a formatted stamp — and this is a correction of a -#: precedent, not a preference. `user_tables.create` writes -#: `datetime.now().strftime('%Y-%m-%dT%H:%M:%S')`: naive LOCAL time, no offset. A browser parses -#: that string as its OWN local time, so on a UTC host read by a non-UTC reader "opened 30 -#: minutes ago" renders as "opened 7 hours ago" and the Today / Past-7-days buckets misfile — -#: with nothing to go red, because both ends are internally consistent. An integer instant has -#: no such reading. (D-18 made the same correction for notification stamps AFTER the defect -#: shipped; this is that lesson applied before it.) -def _now() -> int: - return int(time.time()) - - -def _clean_nav_prefs(raw, page_keys, *, keep_unknown_ut=False): - """Validated wholesale replacement, the `clean_folders` posture: prune, never invent. - - `page_keys` is the set of TOP-LEVEL keys this session may see; a placement of an unknown - or invisible key is dropped (it can return when the grant does — placement is cosmetic, - so pruning is loss-free). Unknown folder refs drop the placement, not the folder. - - `keep_unknown_ut` is the STORE-BLIP escape hatch — see `_placeable_top_keys`. When the - `ut_*` listing could not be read, a `ut_` key absent from `page_keys` is KEPT rather than - pruned: keeping a placement whose database may since have been deleted is cosmetically - harmless, while pruning a live one is the silent data loss this function just stopped - causing. Never widened to non-`ut_` keys — those come from the compiled registry, which - cannot fail to enumerate. - """ - raw = raw if isinstance(raw, dict) else {} - folders, seen = [], set() - for f in (raw.get("folders") or [])[:_MAX_NAV_FOLDERS]: - if not isinstance(f, dict): - continue - fid = str(f.get("id") or "").strip()[:40] - name = " ".join(str(f.get("name") or "").split())[:40] - if not fid or fid in seen or not name: - continue - seen.add(fid) - folders.append({"id": fid, "name": name}) - ids = {f["id"] for f in folders} - placement = {} - src = raw.get("placement") - if isinstance(src, dict): - for k, v in src.items(): - k, v = str(k)[:60], str(v)[:40] - if v not in ids: - continue # the folder is gone; the placement goes with it - if page_keys is None or k in page_keys: - placement[k] = v - elif keep_unknown_ut and k.startswith("ut_"): - placement[k] = v - return {"folders": folders, "placement": placement} - - -def _placeable_top_keys(session): - """`(keys, enumerated)` — the keys a placement may name, INCLUDING this tenant's databases. - - ⛔ THIS IS THE ITEM-6 FIX (wave 27, contract C1), and the bug it closes was invisible by - construction. The old version read `perms.nav_pages` alone — the compiled REGISTRY — and - `perms.py` contains no `ut_` or `user_tables` reference at all, because a tenant's - user-created databases are merged into the nav payload by `nav()` BELOW the permission wall. - So every `ut_*` key was absent from this set, and `_clean_nav_prefs` pruned every placement - naming one — on READ and on WRITE. - - The symptom was "a database I drag into a folder falls out of it later", never "it does not - work": the write answered **200 OK** having stored `{}`, the client kept its optimistic copy - (`Shell.commitPrefs` reverts only on `!ok`), and the folder held for the rest of the session. - The next reload served the pruned copy and the database was back at root. Built-in modules - (`customer_data`, `product_data`) ARE in the registry and persisted fine, which is exactly - why it read as intermittent rather than as a missing feature. - - ⚠ THE SAME MISTAKE, ALREADY MADE AND ALREADY FIXED ONE FUNCTION AWAY. `nav()`'s recents - block (see its `allowed` comment) hit this in wave 23 and solved it by pruning against the - ASSEMBLED page list. This is that lesson applied to the second consumer, which the wave-23 - fix did not reach. - - `enumerated` is False when the `ut_` listing raised — the caller must then NOT treat this set - as authoritative for `ut_` keys (`keep_unknown_ut`). Pruning a person's whole arrangement - because the store blinked would be the original defect wearing a different cause. - """ - pages = perms.nav_pages(session.user) or [] - keys = {p["key"] for p in pages if not p.get("parent")} - try: - import core.user_tables as user_tables - # The SAME listing `nav()` renders from — `may_open`-filtered, so a placement can never - # name a database this session cannot see, and the nav and this door agree by sharing - # one predicate rather than by two lists being kept in step. - # ⭐ W31-T10 (C1/D-175): `nav_entries` lends its own read to `may_open`, so this door is - # ONE document copy rather than `1 + N`. It is not a footnote here — `Shell.tsx` fires - # `/nav/prefs` CONCURRENTLY with `/nav` on the same `Store._lock`, and it measured - # **8,807 ms live / 17,945 ms in-process** on tenant #0, i.e. roughly half the wait the - # owner reports as "the Automation row arrives ten seconds late". - for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin, - st=session.runtime): - keys.add(str(e.get("key") or "")) - except Exception: - return keys, False - return keys, True - - -def _clean_icon(raw): - """A stored/incoming icon, or None. Prune, never invent — `clean_folders`' posture.""" - if not isinstance(raw, dict): - return None - shape, tone = raw.get("shape"), raw.get("tone") - if shape not in _ICON_SHAPES or tone not in _ICON_TONES: - return None - return {"shape": shape, "tone": tone} - - -def _read_nav_meta(runtime): - """The tenant's `nav_meta`, validated on the way OUT as well as in. - - Re-validating a read looks redundant and is not: the bucket outlives this code, an older - build may have written a shape this one no longer accepts, and the whitelist is mirrored - from a vocabulary that lives in another language. A row whose icon does not survive - validation renders the default mark — never a crash, and never a half-drawn glyph. - """ - try: - stored = runtime.get(_NAV_META_KEY) or {} - except Exception: - return {} # a store blip must not take the nav down with it - if not isinstance(stored, dict): - return {} - out = {} - for key, meta in stored.items(): - if not isinstance(meta, dict): - continue - entry = {} - icon = _clean_icon(meta.get("icon")) - if icon: - entry["icon"] = icon - # ⛔ THE `ut_` RULE IS RE-APPLIED ON READ, not trusted from the record. The write - # route refuses a name for a built-in key, but a record written by an older build (or - # by hand) could still carry one — and honouring it would let the store rename - # `customer_data` on screen while `core/registry.py` and every other reader went on - # calling it Customer. A name that could not be written today is not served today. - name = meta.get("name") - if str(key).startswith("ut_") and isinstance(name, str) and name.strip(): - entry["name"] = " ".join(name.split())[:_MAX_NAV_NAME] - if entry: - out[str(key)] = entry - return out - - -def _read_recents(runtime, uname, allowed): - """This user's recents, newest first, PRUNED to what they may currently see. - - ⚠ PRUNED ON READ, never on write, and both halves of that are deliberate: - - · the WRITE happens on every route open — it is the one hot path this file has — so it - must not build the whole nav to validate one key; - · a key whose grant was REVOKED must stop being offered without anyone running a - migration, and must come back if the grant does. That is `_clean_nav_prefs`' own - prune-never-invent posture, applied to a second bucket for the same reason. - - A key that is not in `allowed` therefore reaches the store and never reaches a screen — - which also means a stuffed key cannot be used to discover what exists: it comes back only - if the session could already see it. - """ - try: - stored = (runtime.get(_NAV_RECENTS_KEY) or {}).get(uname) or {} - except Exception: - return [] # a store blip must not take the nav down with it - if not isinstance(stored, dict): - return [] - out = [] - for key, at in stored.items(): - key = str(key) - if allowed is not None and key not in allowed: - continue - try: - at = int(at) - except (TypeError, ValueError): - continue # a stamp this build cannot read is not a stamp - if at <= 0: - continue - out.append({"key": key, "at": at}) - out.sort(key=lambda r: r["at"], reverse=True) - return out[:_MAX_RECENTS] - - -# ── VIEW READERS — shared with `routes_starred` (W35-T39/T42) ──────────────────────────────── -# -# ⛔⛔ W35-T43 (rulings R4/R7) — THE "MARK IMPORTANT" COUNTS ARE GONE FROM THIS ROUTE, and the -# block that computed them (`_important_counts`) went with them. It read one store bucket PER -# DATABASE on the route D-175 spent a whole wave reducing to a single read: MEASURED on a 13-database -# fixture, `GET /nav` performed **14 per-database view-bucket reads** and stamped `important` on -# **14** pages. It is **0 and 0** now. -# -# Those reads were bounded by a 2.5 s wall clock (`_IMPORTANT_BUDGET_S`) whose own note conceded the -# cost: twelve buckets are 6.2 ms WARM and **7,331 ms COLD**, so the budget existed to stop a cold -# container converting a slow success into a manufactured failure against `nav.ts`'s 20 s deadline. -# A budget that can be exhausted is a number that can be silently short, which is why it also had to -# publish a `degraded` entry. R7 removes the whole apparatus by moving the question to -# `GET /starred/counts`, called AFTER the page paints — where a slow answer costs a late badge -# instead of a late rail. Closes D-288 and D-289. -# -# ⚠ WHAT SURVIVES AND WHY: `_visible_views`, `_view_record_count` and `_granted_view_ids` are the -# READERS, and `routes_starred` calls all three. They were never the cost — the per-database store -# read was — and duplicating them into the new route would have put two answers behind one badge. - -#: Suffix `core.view_templates.workspace_key` appends. Stripping it is how a page key becomes the -#: TOPIC key the cohort bucket is named from — `customer_data` -> `customer_table_workspace` -> -#: `customer` -> `customer_cohorts`. Derived rather than re-listed on purpose: `_WS_KEYS` is -#: already the one place `customer_data`/`product_data` are mapped to their topic, and a second -#: copy here would be free to drift from it ([[one-question-two-normalizers]]). -_WS_SUFFIX = "_table_workspace" - - -def _visible_views(doc, uname, is_admin, granted_ids): - """Every view on ONE database that THIS caller can see, from an already-read workspace doc. - - ⛔ PER-CALLER, NOT TENANT-WIDE, and this is the half a server-side count is most likely to get - wrong. `_table_workspace` is `{username: {views, fields, overlays}, '__shared__': {...}}` - — one home per view, never both — so "how many views are marked important here" has a - DIFFERENT answer for every account. Measured on tenant #0: `leadership` has one marked view and - `admin` has none, on the same database. A tenant-wide count would put a number in the owner's - rail that no view sidebar they can open would ever add up to. - - ⚠ `table_store._may_see` is imported rather than re-expressed. It is private, and reaching for - it is still the right call: the alternative is `TableOps.shared_views`, which re-reads the whole - bucket per database (the N whole reads this block exists to avoid), and the only other option is - a second copy of a permission predicate. A wrong copy of `_may_see` widens what a user is told - exists; a private import cannot. - """ - import core.table_store as table_store - out = {} - for vid, view in ((doc.get(uname) or {}).get("views") or {}).items(): - if isinstance(view, dict): - out[str(vid)] = view - for vid, view in ((doc.get(table_store.SHARED_KEY) or {}).get("views") or {}).items(): - if isinstance(view, dict) and table_store._may_see(view, uname, is_admin): - out.setdefault(str(vid), view) - # The wave-21 named-user grants. The ids come from ONE tenant-wide bucket read once for the - # whole request; the RECORD is already in hand, in whichever stratum its owner keeps it. - if granted_ids: - for stratum, blob in doc.items(): - if stratum == uname or not isinstance(blob, dict): - continue - for vid, view in (blob.get("views") or {}).items(): - if str(vid) in granted_ids and isinstance(view, dict): - out.setdefault(str(vid), view) - return out - - -def _view_record_count(cfg, cohorts): - """How many RECORDS this view resolves to, or None when that cannot be answered for free. - - ⛔ `None` IS AN ANSWER AND IT IS THE IMPORTANT ONE. Two of the three shapes below are exact - because the view CARRIES its row set; the third — an ordinary filtered view — can only be - counted by running its filters over the records, and the records are the one thing this route - must never read (`D-175`/`D-185`: `/nav` is a rows-free projection, and reaching for `rows` - here raises by that projection's own contract). So a filtered view is reported as UNCOUNTED and - the database's `partial` flag says so, which is the whole of R6's second sentence applied to a - badge: a limit that cannot be removed is REPORTED with its cause, never papered over with a - number that is short by an unknown amount. - - ⚠ THIS IS ALSO WHY THE SERVER DOES NOT SIMPLY MIRROR THE CLIENT. `CustomerGrid::alertCounts` - counts a filtered view fine and gives up on a SERVER-WINDOWED one (`D-205`'s `Important 0+`); - this end is the exact inverse — it has no rows at all and no window either. The two are honest - about different halves, which is why `partial` had to be on the wire rather than derived. - """ - if not isinstance(cfg, dict): - return None - # A cohort-locked view IS its cohort: the lock and the id are the same fact (`grid_events` - # re-stamps it on every write), and a cohort's membership is a stored pid LIST, not a query. - lock = str(cfg.get("cohortLock") or "").strip() - if lock: - n = cohorts.get(lock) - return int(n) if isinstance(n, int) else None - # A curated row set carries its own count. - pids = cfg.get("memberPids") - if isinstance(pids, list) and pids: - return len(pids) - return None - - -def _visible_database_keys(session): - """`(keys, enumerated)` — every DATABASE this session may open that HAS a view bucket. - - ⭐ ADDED BY W35-T39 because neither existing enumeration in this file answers this question: - - · `_placeable_top_keys` applies the ACCOUNT grant and `may_open`, and **not the TENANT - CATALOGUE** — correctly, because a folder placement is cosmetic and pruning one is loss. - A star scan cannot borrow that: it would open the view bucket of a database this workspace's - catalogue does not include. `nav()` applies the catalogue filter; that helper never has. - · `nav()`'s own `_page_keys` is assembled from the page DICTS it is building for the wire, so - it cannot be reached from another route without rebuilding the payload. - - So this is the KEY-SET question on its own, with the same three walls `nav()` applies in the same - order: the account grant (`perms.nav_pages`), the TENANT catalogue, and `may_open` for the - tenant's own databases. `view_templates.workspace_key` is the last filter and it is what makes - the answer honest for a caller about to read a view bucket: a module surface with no workspace - (`sales`, `ar`) is not a database and has no views to star. - - ⚠ THE `parent` CLAUSE BELOW IS A NO-OP TODAY AND IS KEPT ONLY TO MIRROR `nav()`. `perms.nav_pages` - excludes `group_only` rows and therefore **emits no `parent` at all** (its own docstring says so: - a dangling reference would be worse than a flat list), so `customer_data` arrives here FLAT even - though the registry gives it `parent: 'customers'`. Stated because the opposite is the obvious - reading — this ticket's first draft assumed the clause was excluding children and wrote a - docstring around a defect that does not exist. - - ⚠ `enumerated` is False when the `ut_` listing raised — the caller must not treat the set as - authoritative for `ut_` keys, exactly as `_placeable_top_keys` requires. - """ - import core.view_templates as view_templates - keys, enumerated = set(), True - tcfg = getattr(session.runtime.tenant, "config", None) or {} - tmods = tcfg.get("modules", "all") - allowed = None if tmods == "all" else {str(k) for k in (tmods or [])} - for p in (perms.nav_pages(session.user) or []): - key = str(p.get("key") or "") - if allowed is not None and key not in allowed \ - and str(p.get("parent") or "") not in allowed: - continue - if view_templates.workspace_key(key): - keys.add(key) - try: - import core.user_tables as user_tables - # The SAME `may_open`-filtered listing `nav()` renders from, over the ROWS-FREE projection - # (W32-T02) — so this costs ~0.1% of the document rather than a 28.6 MB deep copy. - for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin, - st=session.runtime): - k = str(e.get("key") or "") - if k: - keys.add(k) - except Exception: - enumerated = False - return keys, enumerated - - -def _granted_view_ids(session): - """Every view id a wave-21 NAMED-USER GRANT lets this caller see. Fail-closed to `set()`. - - ⭐ EXTRACTED (W35-T39) SO THE STAR AND THE COUNT SHARE ONE ANSWER. `_visible_views` above - takes this set as its third stratum, and it had exactly one caller (`_important_counts`) with - the derivation inline — which W35-T43 deletes. `routes_starred` needs the identical set to - answer "which views has this person starred", so the derivation moves here beside the reader - it feeds rather than being copied into a second file ([[one-evaluator-per-question]]). - - ⚠ THE ROLE FILTER IS NOT BELT-AND-BRACES. `shared_with` answers "is there an entry naming - me", and `grid_events._granted_views` — the reader whose answer any consumer of this has to - agree with — then requires `role_for(...) in ('view','edit')`. Dropping that second test - would admit a view the sidebar does not list. One tenant-wide bucket, read once per call, so - it costs a dict lookup per id. - ⚠ Fail-closed: no grants is a NARROWER answer, never a wider one. - """ - import core.shares as shares - try: - return {str(v) for v in - ((shares.shared_with(session.uname, kind="view", st=session.runtime) or {}) - .get("view") or []) - if shares.role_for("view", str(v), session.uname, - st=session.runtime) in ("view", "edit")} - except Exception: - return set() - - -@router.get("/nav") -def nav(session: Session = Depends(require_session)): - """`{pages: [{key,label,source?,chrome}], landing}`. - - Note what this does NOT do: it never returns an empty `pages` list as a way of saying "you - are not allowed". A session that may open nothing at all is a misconfigured account, and it - gets an explicit 403 — an empty 200 is indistinguishable from "the registry is empty" and is - how a permission bug hides in plain sight (X2's never-an-empty-200 rule). - """ - pages = list(perms.nav_pages(session.user) or []) - # Wave 18 C1-TENANT: the REGISTRY is the product's module catalogue — tenant #0's world. - # A tenant record may enable a subset (`modules: [...]`); absent/'all' means everything - # (royal-imports and the compiled builders). A blank tenant enables NOTHING: its nav is - # its own databases, which is what "different set of databases at later waves" means. - tcfg = getattr(session.runtime.tenant, "config", None) or {} - tmods = tcfg.get("modules", "all") - # ⭐⭐ W31-T11 (owner item 6b) — WHAT THE CATALOGUE FILTER REMOVED, SAID OUT LOUD. - # - # Every provisioned tenant carries a restricted list today (`gtmlab`/`loopable`/`nurilab` all - # `['analyst','automation']` — census in `proto/nav-omission-census.md`), so this filter runs - # on every request that is not tenant #0's. It is the INTENDED catalogue, not a defect. But a - # row it removes and a row a store failure dropped are the SAME absence on the wire, and the - # client renders `null` for both — pixel-identical to still-loading. Naming the removals is - # what lets the shell tell "not part of this workspace" from "we could not read it". - _omitted = [] - if tmods != "all": - allowed = {str(k) for k in (tmods or [])} - _omitted = sorted({str(p.get("key")) for p in pages - if p.get("key") not in allowed - and (p.get("parent") or "") not in allowed}) - pages = [p for p in pages - if p.get("key") in allowed or (p.get("parent") or "") in allowed] - # Wave 18 C3-UT: this tenant's user-created databases, merged AFTER the registry rows — - # the host does the same at app.py:7796. Filtered by the per-table wall (`may_open`), so - # the nav cannot offer a row the table routes would refuse. - # ⭐⭐ W31-T10 (contract C1, D-175) — THE DOCUMENT IS READ ONCE FOR THE WHOLE REQUEST. - # - # This route was `2 + N` full deep copies of a 35.8 MB-ceiling document: `nav_entries` took one - # and then `may_open` took another PER TABLE, and the `manage`/`canDelete`/`locked` loop below - # took a SECOND independent one. Median `GET /nav` on tenant #0: **9,548 ms live, 24,741 ms - # in-process** — the second figure is the one that matters, because with the network gone and - # the document resident this route and `/nav/prefs` are the ONLY two that stay slow while every - # other collapses to 20–161 ms. That is per-call CPU, and this is it. - # ⚠ `_ut_defs` is read HERE, above the merge, so the two former reads become one; the locked/ - # manage loop below consumes this same dict. - _ut_defs, _ut_locked_mode, _degraded = {}, "", [] - try: - import core.user_tables as user_tables - # ⭐⭐ W32-T02 (R8/D-175/D-185) — AND THAT ONE READ IS A PROJECTION NOW. The block below - # reads `label`, `source`, `createdBy` and `recordMode`; `may_open` reads `createdBy` and - # the shares registry. Nothing on this route has ever opened a row — and on tenant #0 the - # rows are **99.89% of the document** (28,551,441 bytes; the definitions are 31,220 of - # them). Measured: `all_tables` 1,750 ms -> `all_defs` 1.4 ms, and `GET /nav` 1,921 ms - # median -> see the ticket. THAT is owner item 5: the rail rows did not arrive seconds - # apart because of CSS or ordering, they arrived when their route finished copying. - # ⛔ `all_defs` REFUSES `rows` rather than answering `{}` — if you add something here that - # needs a row, it raises with the reason instead of painting an empty grid. Use - # `all_tables` for that, and know you are buying the whole copy back. - _ut_defs = user_tables.all_defs(st=session.runtime) or {} - # ⛔ NEVER DEFAULT THIS TO `None`. `_ut_defs.get(k)` is `{}` for an unknown key, so its - # `.get("recordMode")` is None too — and `None == None` would mark EVERY database locked - # the moment this import failed. A sentinel that can equal real data is not a sentinel. - _ut_locked_mode = str(user_tables.AUTOMATION_RECORD_MODE) - _lent = user_tables.lend(session.runtime, **{user_tables.STORE_KEY: _ut_defs}) - # ⛔ AND THE EDITOR'S DENY REACHES THIS HALF TOO. `nav_entries` is `may_open`-filtered - # — creator, admin or a share — which knows nothing about the access toggle in - # Manage user. An administrator unticking a shared database left it on the rail. - # `nav_may_open` composes the two and cannot WIDEN past `may_open` on a `ut_*` key. - import core.perm_scope as _perm_scope - # ⭐⭐ WAVE 37 · T07 (D-267, PRD amendment A8) — A RETIRED KEY CANNOT REACH THE NAV, WHATEVER - # THE TENANT DOCUMENT STILL CARRIES. - # - # W33-T44 retired `ut_odoo_customers` / `ut_odoo_products` in favour of `customer_data` / - # `product_data` — ONE SUBJECT, ONE DATABASE — and `apply_plan` pops them from a tenant's - # document on the next rebuild. The owner then reported *"Odoo products"* TWICE, and a live - # label census returned `{'Odoo products': 2}`: the retirement MECHANISM was correct and the - # PATH DID NOT RUN, for causes booked separately (D-251's `frozen(rt)` early return, and the - # rebuild being scoped to `is_royal`). - # - # ⛔ SO THIS IS A READ-SIDE GUARD, NOT THE MIGRATION, AND THE DIFFERENCE IS STATED RATHER THAN - # BLURRED: the stale ROW may still sit in a tenant's document, and removing it is - # `apply_plan`'s job in `odoo_relational.py`. What this line guarantees is that a document - # which still carries one cannot put a SECOND "Odoo products" on somebody's rail while that - # migration has not reached them. A defect whose fix depends on a migration running is a - # defect that comes back on the one tenant the migration missed - # [[a-migration-that-runs-on-the-next-write]]. - # - # ⚠ `RETIRED_KEYS` is IMPORTED, never re-typed here. A hand-copied tuple is exactly the drift - # this guard exists to prevent, and it would go stale the first time a third key is retired. - # ⚠ Inside the same `try:` as the rest of the user-table read on purpose — if the import - # fails, the block's own `except` marks the nav DEGRADED, which is the honest outcome. A - # guard that silently stops guarding is worse than one that says it could not run. - try: - from odoo_relational import RETIRED_KEYS as _retired - except Exception: - _retired = () - for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin, - st=_lent): - if e["key"] in _retired: - continue - if not _perm_scope.nav_may_open(session.user, e["key"], st=_lent): - continue - pages.append({"key": e["key"], "label": e["label"], - "source": e.get("source") or "Blank", "chrome": "main"}) - except Exception: - # ⭐⭐ W31-T11 (owner item 6b) — STILL SWALLOWED, NO LONGER SILENT. - # - # A store blip must not take the whole nav down with it, and that half stands. What - # changed is that it used to answer **200 OK with every database missing** and say - # nothing — so a client cannot tell "this tenant has no databases" from "we could not - # read them", and the rail renders the same complete-looking thing either way. That is - # the owner's *"Connectors and Automation module still disappears"* class of report: - # the payload is a claim about what exists, and a claim it could not verify has to be - # marked as such. `degraded` is that mark; the shell renders it in the affected slot. - _degraded.append("databases") - pass - if not pages: - if tmods != "all": - # A provisioned tenant with no modules and no databases YET is a legitimate empty - # state, not a misconfigured account — the client renders "create your first - # database", and X2's never-an-empty-200 rule is honoured by saying WHY it is - # empty rather than leaving 200-[] ambiguous. - return {"pages": [], "landing": None, "empty": "no_databases"} - raise err(403, "no_surfaces", - "your account has no dashboards assigned. Ask an administrator.") - # WAVE 19 (R8 / C1): the tenant's name + icon overrides, merged LAST — after the registry - # rows, after the tenant module filter, after the user tables. Merged HERE rather than - # applied by the client for one reason: `label` is what every reader of this payload shows, - # including the Settings modal's `moduleLabels` map, so a client-side merge would put the - # override in one door and the registry label in the other. - # - # ⛔ THIS MUTATES `pages` IN PLACE, WHICH IS SAFE ONLY BECAUSE `perms.nav_pages` BUILDS A - # FRESH `row = {...}` PER CALL (perms.py:194) and the ut_ rows are built fresh here. If - # either ever starts handing back cached or module-level dicts, this loop would write one - # tenant's chosen label into the object the NEXT tenant's request reads — a cross-tenant - # leak with no symptom until two tenants rename the same registry key. Copy the rows before - # merging on the day that invariant changes. - meta = _read_nav_meta(session.runtime) - # ⛔⛔ W35-T43 (rulings R4/R7) — THE MARK-IMPORTANT COUNTS ARE NO LONGER COMPUTED HERE. - # - # W34-T10 put them on this payload with a 2.5 s budget and a `degraded` entry, spent in RECENTS - # ORDER so a cold container would reach the databases somebody actually works in first. Every - # one of those was a correct answer to the wrong question: the block was `N` per-database store - # reads on the route D-175 spent a wave reducing to ONE, and no ordering makes an N-read block - # free. MEASURED on a 13-database fixture: **14 view-bucket reads and 14 `important` stamps per - # `/nav`** before this ticket, **0 and 0** after. - # - # R7 moves the question to `GET /starred/counts`, called AFTER the page paints. A slow answer - # there is a late badge; a slow answer here was a late RAIL. Closes D-288 and D-289. - # ⚠ `recents` STAYS and is still ONE read — it is the Home landing's own payload (C10/R7 of wave - # 23), and it was only computed this early so the deleted budget could spend itself in its order. - _page_keys = {str(p.get("key", "")) for p in pages} - recents = _read_recents(session.runtime, session.uname, _page_keys) - # Wave 21 (C3): the definitions, once — `manage`/`canDelete` below answer from `createdBy`. - # WAVE 27 (C9): `locked` answers from `recordMode`, off the same one read. - # ⭐ W31-T10: that read is now the SAME one the merge above did — it used to be a second, - # independent `all_tables`, which is why D-175 called this route `2 + N` rather than `1 + N`. - # ⚠ The fail-closed defaults still hold: both are initialised before the try above, so an - # import or store failure leaves `_ut_locked_mode` empty and nothing is marked locked. - for p in pages: - # `manage` (R14): may THIS session change this row's icon/name? Answered HERE because - # the server is the only end that knows — the client cannot see who created a user - # table. Additive and FAIL-CLOSED (absent reads as "no"), so the rail offers a control - # only where the write would actually land, and the route re-checks it regardless. - # - # ⛔ WAVE 21 (C3/W-5): the "presence IS the answer" shortcut DIED in wave 20 — the - # de5037f share-grant admission widened `may_open`, so a row's presence now includes - # databases merely SHARED to this viewer. `manage` (rename/icon) and `canDelete` are - # therefore answered from the DEFINITION: creator-or-admin strictly, matching the - # walls the PATCH and DELETE routes actually enforce. A grantee sees the row and no - # controls — the honest shape (the old code offered rename to users the route 403'd). - key = str(p.get("key", "")) - if key.startswith("ut_"): - _creator = str((_ut_defs.get(key) or {}).get("createdBy") or "") - _mine = bool(session.admin or (_creator and _creator == session.uname)) - p["manage"] = _mine - p["canDelete"] = _mine - # ⭐ WAVE 27 item 3 (contract C9) — A LOCKED DATABASE SAYS SO IN THE RAIL. - # - # "Locked" is the owner's item-4 vocabulary and it means exactly ONE thing (DESIGN.md - # §4, THE THREE LOCKS): RECORDS cannot be added, deleted or edited — **fields still - # can**. The automation-owned IG child datasets (posts, comments, snapshots) are the - # live example; their rows arrive from the engine, so a "+" row there could only - # refuse, which R8 calls a fake affordance. - # - # ⚠ THE AUTHORITY IS `user_tables.records_mutable`, NOT THIS LINE. The comparison is - # inlined only because `_ut_defs` is already in hand — calling the predicate per row - # would be one store read per database on every nav request — and the VALUE comes - # from the module's own constant rather than a copied string, so the two cannot drift - # to different answers. If that predicate ever grows a second condition, this must - # become a call. - # - # ⚠ ABSENT READS AS UNLOCKED, and that is the safe direction here even though it is - # the opposite of `manage`'s fail-closed: the lock ICON is an affordance hint, while - # the actual refusal is `routes_tables._records_or_refuse`'s 403. A store blip costs - # a missing hint, never a write that should not have landed. - if (_ut_locked_mode - and (_ut_defs.get(key) or {}).get("recordMode") == _ut_locked_mode): - p["locked"] = True - elif session.admin: - p["manage"] = True - # ⛔ W35-T43 — `important` IS NO LONGER STAMPED HERE. W34-T10 emitted it for every database - # this route could read, including `{marked: 0, counted: 0, partial: false}`, so a consumer - # never had to test for the key. R4 retires the count from the rail and the flyout entirely - # (A's W35-T07 is the client half) and R7 moves the numbers to `GET /starred/counts`. - # ⚠ `nav.ts::NavPage` still DECLARES `important` on the client until A's ticket lands; an - # absent key reads as `undefined` there, which is the same thing the optional field already - # meant for a non-database row. Flagged to A rather than assumed harmless. - entry = meta.get(p.get("key")) if meta else None - if not entry: - continue - if entry.get("icon"): - p["icon"] = entry["icon"] - if entry.get("name"): - p["label"] = entry["name"] - landing = perms.landing_page(session.user) - if landing and not any(p.get("key") == landing for p in pages): - landing = pages[0].get("key") - # WAVE 23 (C10 / R7) — the Home landing's recents, on the payload the client already asks - # for. A second round trip for a list this short, computed from a bucket this route is - # already holding the store open for, would be a request per page load for no gain. - # - # ⛔ `allowed` IS THE ASSEMBLED PAGE LIST — the rows this route just built, `ut_*` databases - # included. Pruning against `perms.nav_pages` alone would silently drop every user database - # from Home's recents: the exact surface R7 is about, invisible, with every gate green. - # - # ⚠ WAVE 27 (item 6): the OTHER consumer of that narrower set — the folder-placement door — - # had the identical bug and nobody connected the two for four waves. It is fixed at the - # source now (`_placeable_top_keys`, which merges the same `may_open`-filtered listing), so - # both doors finally agree on what a placeable key is. This block keeps using the assembled - # list because it already holds it: re-enumerating here would be a second store read for an - # answer sitting in a local variable. - # - # ⚠ W34-T10 MOVED THE READ, NOT THE RULE. `recents` is now computed ABOVE the enrichment loop, - # because the important-count block spends its budget in RECENTS ORDER (see there). It is still - # ONE read of `nav_recents`, still pruned against the assembled page list, and it is used here - # unchanged — a second `_read_recents` call would be the extra store read this comment forbids. - # ⭐ W31-T11 — TWO KINDS OF ABSENCE, NAMED SEPARATELY, and both keys are ALWAYS PRESENT. - # `omitted` — this workspace's catalogue does not include these modules. Deliberate. - # `degraded` — a part of this payload could not be read. NOT deliberate, and the shell says - # so in the affected slot instead of rendering a confident nothing. - # ⚠ A key a consumer has to test for is a key a consumer forgets to test for; both ship as - # `[]` rather than being omitted when empty, which is the same rule `limits` follows. - return {"pages": pages, "landing": landing, "recents": recents, - "omitted": _omitted, "degraded": _degraded} - - -@router.post("/nav/opened") -def nav_opened(body: dict = Body(default=None), - session: Session = Depends(require_session)): - """WAVE 23 (C10) — stamp a page as JUST OPENED. Fire-and-forget from the client. - - The client calls this on every route commit, so this is the only write in this file on a - hot path, and three things follow from that: - - · `flush='async'` — the coalescing mode ([[store-async-flush]]). A blocking upload per - page open against an HF-Dataset-backed store would put a network round trip inside every - navigation. `nav_prefs`/`nav_meta` stay `sync` because a folder rename is not a hot path; - this is. - · NO VALIDATION OF THE KEY against the nav. Building the page list to check one string - would make the stamp cost more than the navigation that triggered it — and it would buy - nothing, because the READ prunes to what the session may currently see. An unknown or - revoked key is stored and never served back. - · THE MAP IS CAPPED HERE TOO. Read-side capping alone would let a hostile or buggy client - grow one user's document without bound; `_MAX_RECENTS` entries survive, oldest first to - go, which is the same rule the read applies. - """ - body = body if isinstance(body, dict) else {} - key = str(body.get("key") or "").strip()[:60] - if not key: - raise err(400, "bad_request", "no page was named") - if not session.runtime.available(): - raise err(503, "store_unavailable", - "the tenant store is unavailable. Nothing was recorded.") - stamp, uname = _now(), session.uname - - def _up(data): - data = data if isinstance(data, dict) else {} - mine = dict(data.get(uname) or {}) if isinstance(data.get(uname), dict) else {} - mine[key] = stamp - if len(mine) > _MAX_RECENTS: - # Oldest first. `int(v)` guarded: a stamp an older build wrote in another shape - # sorts as 0 and is the first thing evicted, which is the right answer for a value - # this route can no longer read. - def _at(item): - try: - return int(item[1]) - except (TypeError, ValueError): - return 0 - mine = dict(sorted(mine.items(), key=_at, reverse=True)[:_MAX_RECENTS]) - data[uname] = mine - return data - - try: - session.runtime.update(_NAV_RECENTS_KEY, _up, flush='async') - except Exception: - raise err(503, "store_unavailable", - "the tenant store refused the write. Nothing was recorded.") - return {"key": key, "at": stamp} - - -@router.post("/nav/meta") -def save_nav_meta(body: dict = Body(default=None), - session: Session = Depends(require_session)): - """WAVE 19 (R8 / C1) — set one database's icon and/or name, tenant-wide. - - A PATCH OF ONE KEY, not the wholesale replace `/nav/prefs` uses two routes up, and the - asymmetry is deliberate. Prefs are one user's complete picture of their own rail, so - replacing the document whole is what makes a deleted folder stay deleted. This bucket is - shared by every admin in the tenant: a wholesale write here means whoever saves last - silently erases what the other one named while their tab was open. - - THREE WALLS, all fail-closed: - · the SESSION must be able to open the key at all (the same predicate the nav uses, so - the rail and this route cannot disagree about what exists); - · the WRITE is admin-only — renaming a database is a change every user in the tenant - sees, which is the definition of an administrative act here; - · `name` is refused outright for a non-`ut_` key. Built-in labels are compiled registry - literals: honouring an override would leave this payload and `core/registry.py` - calling the same module two different things, and the wave-16 lesson is that the - client must not be the place that decides what a payload meant. - - `icon: null` CLEARS. An absent field is untouched — which is what makes a rename and an - icon change two independent writes rather than a race between them. - """ - body = body if isinstance(body, dict) else {} - key = str(body.get("key") or "").strip() - if not key: - raise err(400, "bad_request", "no database was named") - # ── THE WALL (ruling R14, 2026-08-04) ──────────────────────────────────────────────────── - # TWO DOORS, because a `ut_` database and a built-in module are owned by different people. - # - # · `ut_*` — the table's CREATOR or a tenant admin, which is exactly what - # `user_tables.may_open` already means. A database you made is yours to name; requiring - # an admin for that was the C1 consequence B flagged and R14 resolved. `session.require` - # is deliberately NOT used here: it is the MODULE gate and would 403 every ut_ key, - # because a user table is not a module. Same split `/nav/schema/{key}` makes. - # · everything else — admin only, and icon only. There is no owner of `customer_data` to - # defer to, and its label is a compiled registry literal (refused below regardless). - if key.startswith("ut_"): - import core.user_tables as user_tables - if not user_tables.get(key, st=session.runtime) or not user_tables.may_open( - key, session.uname, session.admin, st=session.runtime): - raise err(403, "forbidden", "that database belongs to another user") - else: - if not session.admin: - raise err(403, "forbidden", - "only an administrator can change a built-in database's icon") - session.require(key) - - patch = {} - if "icon" in body: - icon = _clean_icon(body.get("icon")) - if body.get("icon") is not None and icon is None: - # Loud, not silent. A shape this build does not know is a CLIENT that has drifted - # from this whitelist, and answering 200 to a write that stored nothing is how - # that drift stays invisible until a user reports "my icon keeps resetting". - raise err(400, "bad_icon", "that icon is not one of the available shapes and tones") - patch["icon"] = icon - if "name" in body: - if not key.startswith("ut_"): - raise err(400, "name_not_allowed", - "only a database you created can be renamed. This one's name comes " - "from the module registry") - name = " ".join(str(body.get("name") or "").split())[:_MAX_NAV_NAME] - if not name: - raise err(400, "bad_request", "a database needs a name") - patch["name"] = name - if not patch: - raise err(400, "bad_request", "nothing to change") - - if not session.runtime.available(): - raise err(503, "store_unavailable", - "the tenant store is unavailable. Nothing was saved.") - - def _up(data): - data = data if isinstance(data, dict) else {} - entry = dict(data.get(key) or {}) if isinstance(data.get(key), dict) else {} - for field, value in patch.items(): - if value is None: - entry.pop(field, None) # an explicit null CLEARS - else: - entry[field] = value - # An entry with nothing left in it is removed rather than stored empty: the read side - # skips empties anyway, and a bucket that accumulates `{}` per key is a document that - # grows forever and says nothing. - if entry: - data[key] = entry - else: - data.pop(key, None) - return data - - try: - session.runtime.update(_NAV_META_KEY, _up) - except Exception: - raise err(503, "store_unavailable", - "the change was not saved: the store refused the write.") - return {"key": key, "meta": _read_nav_meta(session.runtime).get(key, {})} - - -@router.get("/nav/prefs") -def nav_prefs(session: Session = Depends(require_session)): - """This user's database-list folders, re-validated at serve time against what they may - currently see (a revoked page's placement vanishes with the page, and returns with it).""" - try: - stored = (session.runtime.get(_NAV_PREFS_KEY) or {}).get(session.uname) or {} - except Exception: - stored = {} - keys, enumerated = _placeable_top_keys(session) - return {"prefs": _clean_nav_prefs(stored, keys, keep_unknown_ut=not enumerated)} - - -@router.post("/nav/prefs") -def save_nav_prefs(body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Wholesale replace, like the table folder stratum — the client sent its complete - picture, validated here; a partial merge would resurrect deleted folders forever.""" - keys, enumerated = _placeable_top_keys(session) - clean = _clean_nav_prefs(body or {}, keys, keep_unknown_ut=not enumerated) - if not session.runtime.available(): - raise err(503, "store_unavailable", - "the tenant store is unavailable. Nothing was saved.") - - def _up(data): - data = data if isinstance(data, dict) else {} - if clean["folders"] or clean["placement"]: - data[session.uname] = clean - else: - data.pop(session.uname, None) - return data - - try: - session.runtime.update(_NAV_PREFS_KEY, _up) - except Exception: - raise err(503, "store_unavailable", - "the folder change was not saved: the store refused the write.") - return {"prefs": clean} - - -@router.get("/nav/schema/{key}") -def nav_schema(key: str, session: Session = Depends(require_session)): - """The database's schema drawer payload: its field contract + the semantic measures this - session may build with. - - TWO WALLS, ANSWERING TWO QUESTIONS, AND THEY FAIL DIFFERENTLY ON PURPOSE. The DATABASE wall - is the same predicate the nav uses — a key this session may not open answers 403, never a - redacted schema. The FIELD wall runs *after* it, on a database this session may open, and - its whole job is to narrow: the drawer serves the columns this reader receives everywhere - else, and no more. - - ⭐⭐ W38-T18 — THE FIELD WALL IS NEW HERE AND THIS DOOR IS WHERE IT WAS MISSING. Every grid - door has stripped hidden columns from both wires since W36-T21; the schema drawer beside them - served the raw contract to anyone `require`/`may_open` admitted. Owner instruction 10 — - *"Metrics field lookback and value must abide by the Filter permissioning"* — is satisfied on - the picker side already (the condition kit takes `fields` as a PROP off a payload that is - walled upstream), so this route is the one place the same reader could still learn the name, - the label and the VALUE VOCABULARY of a column their grid refuses them. - """ - # Wave 18 C3-UT: a user table's schema is its own definition, walled by ITS predicate - # (`may_open`) rather than the module grant machinery — `session.require` would 403 every - # ut key because a user table is deliberately not a module. - if key.startswith("ut_"): - import core.user_tables as user_tables - defn = user_tables.get(key, st=session.runtime) - if not defn or not user_tables.may_open(key, session.uname, session.admin, - st=session.runtime): - raise err(403, "forbidden", "that database belongs to another user") - # ⭐⭐ W38-T18 — THE FIELD WALL ON A `ut_*` DATABASE, WHICH THIS DOOR HAS NEVER RUN. - # `may_open` answers *IF* you reach the database and says nothing about WHICH COLUMNS, so - # this drawer handed `defn["fields"]` whole to every account it admitted. The bypass is - # older than this ticket (it dates to the wall itself); what changed is the CATEGORY it - # leaks — since W38-T16 a hidden `ut_*` column means an administrator's `hiddenFields` - # *or* a column granted away, and a share door whose schema drawer still names the column - # tells the ungranted reader exactly what they were refused. - # ⚠ MERGED BEFORE THE WALL, NEVER AFTER — the same order `routes_tables.ut_assembly` - # runs, and for the reason `_ut_shared_fields` gives one door over: the closure must see - # the WHOLE contract, or a definition field whose formula reads a hidden tenant-wide - # column sits outside its reach and carries that value out wearing a second name. - # ⛔ ONLY THE DEFINITION'S OWN FIELDS ARE RETURNED. This drawer has never published the - # shared stratum and this ticket is not where it starts; widening the list handed to the - # closure can only ever ADD to the hidden set, never remove from it. - import routes_tables as _rt - base = list(defn.get("fields") or []) - hidden = _rt._ut_hidden(session, key, - _rt._ut_shared_fields(session, key, base), - st=session.runtime) - # WAVE 19 (R8) — the drawer wears the RENAMED name. A rail that says one thing and a - # schema panel opened from it that says another is the drift a rename is supposed to - # remove, not create. - return {"key": key, - "label": (_read_nav_meta(session.runtime).get(key, {}).get("name") - or defn.get("label") or key), - "source": defn.get("source") or "Blank", - "fields": [{"key": f["key"], "label": f["label"], "type": f["type"], - "source": f.get("source") or "overlay", - "description": str(f.get("description") or "")} - for f in base if f.get("key") not in hidden], - "measures": []} - session.require(key) - pages = perms.nav_pages(session.user) or [] - page = next((p for p in pages if p.get("key") == key), None) - if page is None: - raise err(404, "unknown_page", f"{key!r} is not a database this session can see") - fields = [] - if key in ("customer_data", "cohort", "customers"): - try: - import aios_grid - import routes_customers as _rc - # ⭐⭐ W38-T18 — THE FIELD WALL ON THE CUSTOMER CONTRACT, AND THE `options` LINE FOUR - # ROWS DOWN IS WHY IT IS A LEAK RATHER THAN AN UNTIDINESS. A schema drawer that names - # a hidden column has told the reader it exists; one that ships `options` has handed - # them the VALUE VOCABULARY of every choice and status column their grid refuses — - # the set of statuses the business uses, read straight off a door with no wall. - # ⛔ `_hidden_for` IS REUSED, NOT RE-DERIVED, and the difference is not cosmetic. - # `hidden_keys(user, MODULE, aios_grid.FIELDS)` is the obvious spelling and it is - # INCOMPLETE: that function's own docstring records that a wall computed from the - # canonical list alone answers "not hidden" for every runtime column, so the - # tenant-wide stratum never reaches the closure and a formula over a hidden column - # comes back. One evaluator, one caller, no second idea of what this session hides. - # ⚠ INSIDE THE `try`, DELIBERATELY. If the wall cannot be computed the route already - # answers with an empty contract rather than a full one — fail-closed is the only - # direction a schema door may fail, and the gate's ADMIN control is what stops that - # silent empty being mistaken for a wall that worked. - hidden = _rc._hidden_for(session) - for f in aios_grid.FIELDS: - if f["key"] in hidden: - continue - entry = {"key": f["key"], "label": f["label"], "type": f["type"], - "source": f["source"], - "description": str(f.get("description") or "")} - if f.get("options"): - entry["options"] = list(f["options"]) - fields.append(entry) - except Exception: - fields = [] - measures = [] - try: - from core import measure_resolve - import core.perm_scope as _perm_scope - team_id = perms.scope_team_id(session.user) - # ⭐⭐ W38-T19 — THE METRICS CAPABILITY REACHES THIS DRAWER TOO, AND IT IS THE ONE DOOR - # THE THREE GRID ASSEMBLIES DO NOT SPEAK FOR. This route calls `measure_resolve.offer` - # ITSELF rather than reading a `measures` key off an assembly, so a wall applied in - # `routes_customers` / `routes_products` / `routes_tables` narrows every grid and leaves - # the panel beside them naming the full lookback catalogue — the same one-door-short - # shape W38-T18 fixed for FIELDS in this very function, one key over. - # ⚠ THE UT_ BRANCH ABOVE RETURNS BEFORE THIS AND ALREADY SERVES `[]`, so there is - # nothing to narrow there; if it ever starts publishing an offer, it needs this line. - _offer = ((measure_resolve.offer(team_id) or []) - if _perm_scope.may_metrics(session.user, key) else []) - for m in _offer: - measures.append({"key": str(m.get("key") or ""), - "label": str(m.get("label") or m.get("key") or ""), - "type": str(m.get("type") or "")}) - except Exception: - measures = [] - out = {"key": key, "label": page.get("label") or key, - "source": page.get("source") or "", "fields": fields, "measures": measures} - if not fields: - # Honest, never a mock: a database whose contract is not yet published says so. - out["note"] = "This database has not published a field contract yet." - return out +"""routes_nav.py — X2's `GET /api/v1/nav`: the registry, filtered by what this session may open. + +SERVER-FILTERED, not client-filtered. The client renders what it is given and never decides who +may see what — a nav that hides a link the API would still serve is a UI courtesy, not a +permission. `core.perms.nav_pages` is the single predicate (shared with `may_open`'s page gate), +so the nav and the 403 can never disagree about a grant. + +The full rule set — archived is invisible to everyone, `group_only` rows are excluded so no +`parent` reference can dangle, `nav: False` surfaces ship as `chrome: 'utility'`, and the +`prefs.json` Library preference is deliberately NOT applied — is documented on +`core.perms.nav_pages`, which is where it belongs: one place, both callers. +""" +import time + +from fastapi import APIRouter, Body, Depends + +from deps import Session, err, perms, require_session + +router = APIRouter(prefix="/api/v1") + +#: Wave 2026-08-02 (C-SCHEMA): per-user folders over the database list, the view-folder +#: pattern applied to the nav. Cosmetic per-user state — placement never grants or hides a +#: surface (the server-filtered nav still decides what exists). +_NAV_PREFS_KEY = "nav_prefs" +_MAX_NAV_FOLDERS = 16 + +#: WAVE 19 (R8 / contract C1): the database's NAME and ICON overrides. +#: +#: ⚠ TENANT-WIDE, which is the whole difference from `nav_prefs` above and the reason it is a +#: separate bucket rather than another field in that one. Folders and placement are one +#: person's arrangement of their own rail — per-user by definition. What a database is CALLED +#: and what it looks like are facts about the database: a workspace where two people call the +#: same table different things has no shared vocabulary left to discuss it in. Same store, two +#: buckets, because they answer to two different owners. +#: +#: Shape: {"": {"icon": {"shape": , "tone": }, +#: "name": ""}} +_NAV_META_KEY = "nav_meta" + +#: The grid's folder-icon vocabulary, mirrored — 12 shapes x 5 tones. The CLIENT imports these +#: from `customer-grid/types` rather than redefining them; this end cannot import TypeScript, +#: so it is the one place the list is written twice. +#: +#: ⛔ WHAT AN UNKNOWN VALUE MUST NOT DO IS BE STORED. `FolderMark` indexes its path table by +#: shape and maps the result, so a shape this whitelist let through and the renderer does not +#: know is `undefined.map()` — a blank rail, from a stored preference, for every user in the +#: tenant until somebody edits the store by hand. Refusing at the door is the cheap end of +#: that. If the two lists ever drift the symptom is an icon that silently reverts to the +#: default, which is the loudest SAFE failure available here. +_ICON_SHAPES = frozenset({"folder", "star", "flag", "tag", "bookmark", "grid", + "chart", "map", "users", "clock", "heart", "bolt"}) +_ICON_TONES = frozenset({"neutral", "blue", "green", "yellow", "red"}) +_MAX_NAV_NAME = 60 + +#: WAVE 23 (contract C10 / ruling R7) — the Home landing's RECENTS. +#: +#: PER-USER, like `nav_prefs` two buckets up and unlike `nav_meta`: what I opened last is +#: nobody else's business, and a tenant-wide "recently opened" would be a surveillance feature +#: rather than a convenience one. +#: +#: ⛔ A MAP KEYED BY PAGE, NOT AN APPEND LOG, and the difference is the whole feature. +#: `{username: {pageKey: }}` — re-opening a database OVERWRITES its stamp. An +#: append-only list capped at 50 fills with fifty copies of the same ten databases inside one +#: working session, and the cap then evicts OLDEST-FIRST: the tenth database you touched falls +#: off the list while forty slots hold repeat visits to the first. Keying by page makes +#: "recent" mean what the word means, and makes the cap bound the number of DATABASES +#: remembered rather than the number of clicks. +_NAV_RECENTS_KEY = "nav_recents" +_MAX_RECENTS = 50 + +#: ⚠ EPOCH SECONDS (UTC by definition), never a formatted stamp — and this is a correction of a +#: precedent, not a preference. `user_tables.create` writes +#: `datetime.now().strftime('%Y-%m-%dT%H:%M:%S')`: naive LOCAL time, no offset. A browser parses +#: that string as its OWN local time, so on a UTC host read by a non-UTC reader "opened 30 +#: minutes ago" renders as "opened 7 hours ago" and the Today / Past-7-days buckets misfile — +#: with nothing to go red, because both ends are internally consistent. An integer instant has +#: no such reading. (D-18 made the same correction for notification stamps AFTER the defect +#: shipped; this is that lesson applied before it.) +def _now() -> int: + return int(time.time()) + + +def _clean_nav_prefs(raw, page_keys, *, keep_unknown_ut=False): + """Validated wholesale replacement, the `clean_folders` posture: prune, never invent. + + `page_keys` is the set of TOP-LEVEL keys this session may see; a placement of an unknown + or invisible key is dropped (it can return when the grant does — placement is cosmetic, + so pruning is loss-free). Unknown folder refs drop the placement, not the folder. + + `keep_unknown_ut` is the STORE-BLIP escape hatch — see `_placeable_top_keys`. When the + `ut_*` listing could not be read, a `ut_` key absent from `page_keys` is KEPT rather than + pruned: keeping a placement whose database may since have been deleted is cosmetically + harmless, while pruning a live one is the silent data loss this function just stopped + causing. Never widened to non-`ut_` keys — those come from the compiled registry, which + cannot fail to enumerate. + """ + raw = raw if isinstance(raw, dict) else {} + folders, seen = [], set() + for f in (raw.get("folders") or [])[:_MAX_NAV_FOLDERS]: + if not isinstance(f, dict): + continue + fid = str(f.get("id") or "").strip()[:40] + name = " ".join(str(f.get("name") or "").split())[:40] + if not fid or fid in seen or not name: + continue + seen.add(fid) + folders.append({"id": fid, "name": name}) + ids = {f["id"] for f in folders} + placement = {} + src = raw.get("placement") + if isinstance(src, dict): + for k, v in src.items(): + k, v = str(k)[:60], str(v)[:40] + if v not in ids: + continue # the folder is gone; the placement goes with it + if page_keys is None or k in page_keys: + placement[k] = v + elif keep_unknown_ut and k.startswith("ut_"): + placement[k] = v + return {"folders": folders, "placement": placement} + + +def _placeable_top_keys(session): + """`(keys, enumerated)` — the keys a placement may name, INCLUDING this tenant's databases. + + ⛔ THIS IS THE ITEM-6 FIX (wave 27, contract C1), and the bug it closes was invisible by + construction. The old version read `perms.nav_pages` alone — the compiled REGISTRY — and + `perms.py` contains no `ut_` or `user_tables` reference at all, because a tenant's + user-created databases are merged into the nav payload by `nav()` BELOW the permission wall. + So every `ut_*` key was absent from this set, and `_clean_nav_prefs` pruned every placement + naming one — on READ and on WRITE. + + The symptom was "a database I drag into a folder falls out of it later", never "it does not + work": the write answered **200 OK** having stored `{}`, the client kept its optimistic copy + (`Shell.commitPrefs` reverts only on `!ok`), and the folder held for the rest of the session. + The next reload served the pruned copy and the database was back at root. Built-in modules + (`customer_data`, `product_data`) ARE in the registry and persisted fine, which is exactly + why it read as intermittent rather than as a missing feature. + + ⚠ THE SAME MISTAKE, ALREADY MADE AND ALREADY FIXED ONE FUNCTION AWAY. `nav()`'s recents + block (see its `allowed` comment) hit this in wave 23 and solved it by pruning against the + ASSEMBLED page list. This is that lesson applied to the second consumer, which the wave-23 + fix did not reach. + + `enumerated` is False when the `ut_` listing raised — the caller must then NOT treat this set + as authoritative for `ut_` keys (`keep_unknown_ut`). Pruning a person's whole arrangement + because the store blinked would be the original defect wearing a different cause. + """ + pages = perms.nav_pages(session.user) or [] + keys = {p["key"] for p in pages if not p.get("parent")} + try: + import core.user_tables as user_tables + # The SAME listing `nav()` renders from — `may_open`-filtered, so a placement can never + # name a database this session cannot see, and the nav and this door agree by sharing + # one predicate rather than by two lists being kept in step. + # ⭐ W31-T10 (C1/D-175): `nav_entries` lends its own read to `may_open`, so this door is + # ONE document copy rather than `1 + N`. It is not a footnote here — `Shell.tsx` fires + # `/nav/prefs` CONCURRENTLY with `/nav` on the same `Store._lock`, and it measured + # **8,807 ms live / 17,945 ms in-process** on tenant #0, i.e. roughly half the wait the + # owner reports as "the Automation row arrives ten seconds late". + for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin, + st=session.runtime): + keys.add(str(e.get("key") or "")) + except Exception: + return keys, False + return keys, True + + +def _clean_icon(raw): + """A stored/incoming icon, or None. Prune, never invent — `clean_folders`' posture.""" + if not isinstance(raw, dict): + return None + shape, tone = raw.get("shape"), raw.get("tone") + if shape not in _ICON_SHAPES or tone not in _ICON_TONES: + return None + return {"shape": shape, "tone": tone} + + +def _read_nav_meta(runtime): + """The tenant's `nav_meta`, validated on the way OUT as well as in. + + Re-validating a read looks redundant and is not: the bucket outlives this code, an older + build may have written a shape this one no longer accepts, and the whitelist is mirrored + from a vocabulary that lives in another language. A row whose icon does not survive + validation renders the default mark — never a crash, and never a half-drawn glyph. + """ + try: + stored = runtime.get(_NAV_META_KEY) or {} + except Exception: + return {} # a store blip must not take the nav down with it + if not isinstance(stored, dict): + return {} + out = {} + for key, meta in stored.items(): + if not isinstance(meta, dict): + continue + entry = {} + icon = _clean_icon(meta.get("icon")) + if icon: + entry["icon"] = icon + # ⛔ THE `ut_` RULE IS RE-APPLIED ON READ, not trusted from the record. The write + # route refuses a name for a built-in key, but a record written by an older build (or + # by hand) could still carry one — and honouring it would let the store rename + # `customer_data` on screen while `core/registry.py` and every other reader went on + # calling it Customer. A name that could not be written today is not served today. + name = meta.get("name") + if str(key).startswith("ut_") and isinstance(name, str) and name.strip(): + entry["name"] = " ".join(name.split())[:_MAX_NAV_NAME] + if entry: + out[str(key)] = entry + return out + + +def _read_recents(runtime, uname, allowed): + """This user's recents, newest first, PRUNED to what they may currently see. + + ⚠ PRUNED ON READ, never on write, and both halves of that are deliberate: + + · the WRITE happens on every route open — it is the one hot path this file has — so it + must not build the whole nav to validate one key; + · a key whose grant was REVOKED must stop being offered without anyone running a + migration, and must come back if the grant does. That is `_clean_nav_prefs`' own + prune-never-invent posture, applied to a second bucket for the same reason. + + A key that is not in `allowed` therefore reaches the store and never reaches a screen — + which also means a stuffed key cannot be used to discover what exists: it comes back only + if the session could already see it. + """ + try: + stored = (runtime.get(_NAV_RECENTS_KEY) or {}).get(uname) or {} + except Exception: + return [] # a store blip must not take the nav down with it + if not isinstance(stored, dict): + return [] + out = [] + for key, at in stored.items(): + key = str(key) + if allowed is not None and key not in allowed: + continue + try: + at = int(at) + except (TypeError, ValueError): + continue # a stamp this build cannot read is not a stamp + if at <= 0: + continue + out.append({"key": key, "at": at}) + out.sort(key=lambda r: r["at"], reverse=True) + return out[:_MAX_RECENTS] + + +# ── VIEW READERS — shared with `routes_starred` (W35-T39/T42) ──────────────────────────────── +# +# ⛔⛔ W35-T43 (rulings R4/R7) — THE "MARK IMPORTANT" COUNTS ARE GONE FROM THIS ROUTE, and the +# block that computed them (`_important_counts`) went with them. It read one store bucket PER +# DATABASE on the route D-175 spent a whole wave reducing to a single read: MEASURED on a 13-database +# fixture, `GET /nav` performed **14 per-database view-bucket reads** and stamped `important` on +# **14** pages. It is **0 and 0** now. +# +# Those reads were bounded by a 2.5 s wall clock (`_IMPORTANT_BUDGET_S`) whose own note conceded the +# cost: twelve buckets are 6.2 ms WARM and **7,331 ms COLD**, so the budget existed to stop a cold +# container converting a slow success into a manufactured failure against `nav.ts`'s 20 s deadline. +# A budget that can be exhausted is a number that can be silently short, which is why it also had to +# publish a `degraded` entry. R7 removes the whole apparatus by moving the question to +# `GET /starred/counts`, called AFTER the page paints — where a slow answer costs a late badge +# instead of a late rail. Closes D-288 and D-289. +# +# ⚠ WHAT SURVIVES AND WHY: `_visible_views`, `_view_record_count` and `_granted_view_ids` are the +# READERS, and `routes_starred` calls all three. They were never the cost — the per-database store +# read was — and duplicating them into the new route would have put two answers behind one badge. + +#: Suffix `core.view_templates.workspace_key` appends. Stripping it is how a page key becomes the +#: TOPIC key the cohort bucket is named from — `customer_data` -> `customer_table_workspace` -> +#: `customer` -> `customer_cohorts`. Derived rather than re-listed on purpose: `_WS_KEYS` is +#: already the one place `customer_data`/`product_data` are mapped to their topic, and a second +#: copy here would be free to drift from it ([[one-question-two-normalizers]]). +_WS_SUFFIX = "_table_workspace" + + +def _visible_views(doc, uname, is_admin, granted_ids): + """Every view on ONE database that THIS caller can see, from an already-read workspace doc. + + ⛔ PER-CALLER, NOT TENANT-WIDE, and this is the half a server-side count is most likely to get + wrong. `_table_workspace` is `{username: {views, fields, overlays}, '__shared__': {...}}` + — one home per view, never both — so "how many views are marked important here" has a + DIFFERENT answer for every account. Measured on tenant #0: `leadership` has one marked view and + `admin` has none, on the same database. A tenant-wide count would put a number in the owner's + rail that no view sidebar they can open would ever add up to. + + ⚠ `table_store._may_see` is imported rather than re-expressed. It is private, and reaching for + it is still the right call: the alternative is `TableOps.shared_views`, which re-reads the whole + bucket per database (the N whole reads this block exists to avoid), and the only other option is + a second copy of a permission predicate. A wrong copy of `_may_see` widens what a user is told + exists; a private import cannot. + """ + import core.table_store as table_store + out = {} + for vid, view in ((doc.get(uname) or {}).get("views") or {}).items(): + if isinstance(view, dict): + out[str(vid)] = view + for vid, view in ((doc.get(table_store.SHARED_KEY) or {}).get("views") or {}).items(): + if isinstance(view, dict) and table_store._may_see(view, uname, is_admin): + out.setdefault(str(vid), view) + # The wave-21 named-user grants. The ids come from ONE tenant-wide bucket read once for the + # whole request; the RECORD is already in hand, in whichever stratum its owner keeps it. + if granted_ids: + for stratum, blob in doc.items(): + if stratum == uname or not isinstance(blob, dict): + continue + for vid, view in (blob.get("views") or {}).items(): + if str(vid) in granted_ids and isinstance(view, dict): + out.setdefault(str(vid), view) + return out + + +def _view_record_count(cfg, cohorts): + """How many RECORDS this view resolves to, or None when that cannot be answered for free. + + ⛔ `None` IS AN ANSWER AND IT IS THE IMPORTANT ONE. Two of the three shapes below are exact + because the view CARRIES its row set; the third — an ordinary filtered view — can only be + counted by running its filters over the records, and the records are the one thing this route + must never read (`D-175`/`D-185`: `/nav` is a rows-free projection, and reaching for `rows` + here raises by that projection's own contract). So a filtered view is reported as UNCOUNTED and + the database's `partial` flag says so, which is the whole of R6's second sentence applied to a + badge: a limit that cannot be removed is REPORTED with its cause, never papered over with a + number that is short by an unknown amount. + + ⚠ THIS IS ALSO WHY THE SERVER DOES NOT SIMPLY MIRROR THE CLIENT. `CustomerGrid::alertCounts` + counts a filtered view fine and gives up on a SERVER-WINDOWED one (`D-205`'s `Important 0+`); + this end is the exact inverse — it has no rows at all and no window either. The two are honest + about different halves, which is why `partial` had to be on the wire rather than derived. + """ + if not isinstance(cfg, dict): + return None + # A cohort-locked view IS its cohort: the lock and the id are the same fact (`grid_events` + # re-stamps it on every write), and a cohort's membership is a stored pid LIST, not a query. + lock = str(cfg.get("cohortLock") or "").strip() + if lock: + n = cohorts.get(lock) + return int(n) if isinstance(n, int) else None + # A curated row set carries its own count. + pids = cfg.get("memberPids") + if isinstance(pids, list) and pids: + return len(pids) + return None + + +def _visible_database_keys(session): + """`(keys, enumerated)` — every DATABASE this session may open that HAS a view bucket. + + ⭐ ADDED BY W35-T39 because neither existing enumeration in this file answers this question: + + · `_placeable_top_keys` applies the ACCOUNT grant and `may_open`, and **not the TENANT + CATALOGUE** — correctly, because a folder placement is cosmetic and pruning one is loss. + A star scan cannot borrow that: it would open the view bucket of a database this workspace's + catalogue does not include. `nav()` applies the catalogue filter; that helper never has. + · `nav()`'s own `_page_keys` is assembled from the page DICTS it is building for the wire, so + it cannot be reached from another route without rebuilding the payload. + + So this is the KEY-SET question on its own, with the same three walls `nav()` applies in the same + order: the account grant (`perms.nav_pages`), the TENANT catalogue, and `may_open` for the + tenant's own databases. `view_templates.workspace_key` is the last filter and it is what makes + the answer honest for a caller about to read a view bucket: a module surface with no workspace + (`sales`, `ar`) is not a database and has no views to star. + + ⚠ THE `parent` CLAUSE BELOW IS A NO-OP TODAY AND IS KEPT ONLY TO MIRROR `nav()`. `perms.nav_pages` + excludes `group_only` rows and therefore **emits no `parent` at all** (its own docstring says so: + a dangling reference would be worse than a flat list), so `customer_data` arrives here FLAT even + though the registry gives it `parent: 'customers'`. Stated because the opposite is the obvious + reading — this ticket's first draft assumed the clause was excluding children and wrote a + docstring around a defect that does not exist. + + ⚠ `enumerated` is False when the `ut_` listing raised — the caller must not treat the set as + authoritative for `ut_` keys, exactly as `_placeable_top_keys` requires. + """ + import core.view_templates as view_templates + keys, enumerated = set(), True + tcfg = getattr(session.runtime.tenant, "config", None) or {} + tmods = tcfg.get("modules", "all") + allowed = None if tmods == "all" else {str(k) for k in (tmods or [])} + for p in (perms.nav_pages(session.user) or []): + key = str(p.get("key") or "") + if allowed is not None and key not in allowed \ + and str(p.get("parent") or "") not in allowed: + continue + if view_templates.workspace_key(key): + keys.add(key) + try: + import core.user_tables as user_tables + # The SAME `may_open`-filtered listing `nav()` renders from, over the ROWS-FREE projection + # (W32-T02) — so this costs ~0.1% of the document rather than a 28.6 MB deep copy. + for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin, + st=session.runtime): + k = str(e.get("key") or "") + if k: + keys.add(k) + except Exception: + enumerated = False + return keys, enumerated + + +def _granted_view_ids(session): + """Every view id a wave-21 NAMED-USER GRANT lets this caller see. Fail-closed to `set()`. + + ⭐ EXTRACTED (W35-T39) SO THE STAR AND THE COUNT SHARE ONE ANSWER. `_visible_views` above + takes this set as its third stratum, and it had exactly one caller (`_important_counts`) with + the derivation inline — which W35-T43 deletes. `routes_starred` needs the identical set to + answer "which views has this person starred", so the derivation moves here beside the reader + it feeds rather than being copied into a second file ([[one-evaluator-per-question]]). + + ⚠ THE ROLE FILTER IS NOT BELT-AND-BRACES. `shared_with` answers "is there an entry naming + me", and `grid_events._granted_views` — the reader whose answer any consumer of this has to + agree with — then requires `role_for(...) in ('view','edit')`. Dropping that second test + would admit a view the sidebar does not list. One tenant-wide bucket, read once per call, so + it costs a dict lookup per id. + ⚠ Fail-closed: no grants is a NARROWER answer, never a wider one. + """ + import core.shares as shares + try: + return {str(v) for v in + ((shares.shared_with(session.uname, kind="view", st=session.runtime) or {}) + .get("view") or []) + if shares.role_for("view", str(v), session.uname, + st=session.runtime) in ("view", "edit")} + except Exception: + return set() + + +@router.get("/nav") +def nav(session: Session = Depends(require_session)): + """`{pages: [{key,label,source?,chrome}], landing}`. + + Note what this does NOT do: it never returns an empty `pages` list as a way of saying "you + are not allowed". A session that may open nothing at all is a misconfigured account, and it + gets an explicit 403 — an empty 200 is indistinguishable from "the registry is empty" and is + how a permission bug hides in plain sight (X2's never-an-empty-200 rule). + """ + pages = list(perms.nav_pages(session.user) or []) + # Wave 18 C1-TENANT: the REGISTRY is the product's module catalogue — tenant #0's world. + # A tenant record may enable a subset (`modules: [...]`); absent/'all' means everything + # (royal-imports and the compiled builders). A blank tenant enables NOTHING: its nav is + # its own databases, which is what "different set of databases at later waves" means. + tcfg = getattr(session.runtime.tenant, "config", None) or {} + tmods = tcfg.get("modules", "all") + # ⭐⭐ W31-T11 (owner item 6b) — WHAT THE CATALOGUE FILTER REMOVED, SAID OUT LOUD. + # + # Every provisioned tenant carries a restricted list today (`gtmlab`/`loopable`/`nurilab` all + # `['analyst','automation']` — census in `proto/nav-omission-census.md`), so this filter runs + # on every request that is not tenant #0's. It is the INTENDED catalogue, not a defect. But a + # row it removes and a row a store failure dropped are the SAME absence on the wire, and the + # client renders `null` for both — pixel-identical to still-loading. Naming the removals is + # what lets the shell tell "not part of this workspace" from "we could not read it". + _omitted = [] + if tmods != "all": + allowed = {str(k) for k in (tmods or [])} + _omitted = sorted({str(p.get("key")) for p in pages + if p.get("key") not in allowed + and (p.get("parent") or "") not in allowed}) + pages = [p for p in pages + if p.get("key") in allowed or (p.get("parent") or "") in allowed] + # Wave 18 C3-UT: this tenant's user-created databases, merged AFTER the registry rows — + # the host does the same at app.py:7796. Filtered by the per-table wall (`may_open`), so + # the nav cannot offer a row the table routes would refuse. + # ⭐⭐ W31-T10 (contract C1, D-175) — THE DOCUMENT IS READ ONCE FOR THE WHOLE REQUEST. + # + # This route was `2 + N` full deep copies of a 35.8 MB-ceiling document: `nav_entries` took one + # and then `may_open` took another PER TABLE, and the `manage`/`canDelete`/`locked` loop below + # took a SECOND independent one. Median `GET /nav` on tenant #0: **9,548 ms live, 24,741 ms + # in-process** — the second figure is the one that matters, because with the network gone and + # the document resident this route and `/nav/prefs` are the ONLY two that stay slow while every + # other collapses to 20–161 ms. That is per-call CPU, and this is it. + # ⚠ `_ut_defs` is read HERE, above the merge, so the two former reads become one; the locked/ + # manage loop below consumes this same dict. + _ut_defs, _ut_locked_mode, _degraded = {}, "", [] + try: + import core.user_tables as user_tables + # ⭐⭐ W32-T02 (R8/D-175/D-185) — AND THAT ONE READ IS A PROJECTION NOW. The block below + # reads `label`, `source`, `createdBy` and `recordMode`; `may_open` reads `createdBy` and + # the shares registry. Nothing on this route has ever opened a row — and on tenant #0 the + # rows are **99.89% of the document** (28,551,441 bytes; the definitions are 31,220 of + # them). Measured: `all_tables` 1,750 ms -> `all_defs` 1.4 ms, and `GET /nav` 1,921 ms + # median -> see the ticket. THAT is owner item 5: the rail rows did not arrive seconds + # apart because of CSS or ordering, they arrived when their route finished copying. + # ⛔ `all_defs` REFUSES `rows` rather than answering `{}` — if you add something here that + # needs a row, it raises with the reason instead of painting an empty grid. Use + # `all_tables` for that, and know you are buying the whole copy back. + _ut_defs = user_tables.all_defs(st=session.runtime) or {} + # ⛔ NEVER DEFAULT THIS TO `None`. `_ut_defs.get(k)` is `{}` for an unknown key, so its + # `.get("recordMode")` is None too — and `None == None` would mark EVERY database locked + # the moment this import failed. A sentinel that can equal real data is not a sentinel. + _ut_locked_mode = str(user_tables.AUTOMATION_RECORD_MODE) + _lent = user_tables.lend(session.runtime, **{user_tables.STORE_KEY: _ut_defs}) + # ⛔ AND THE EDITOR'S DENY REACHES THIS HALF TOO. `nav_entries` is `may_open`-filtered + # — creator, admin or a share — which knows nothing about the access toggle in + # Manage user. An administrator unticking a shared database left it on the rail. + # `nav_may_open` composes the two and cannot WIDEN past `may_open` on a `ut_*` key. + import core.perm_scope as _perm_scope + # ⭐⭐ WAVE 37 · T07 (D-267, PRD amendment A8) — A RETIRED KEY CANNOT REACH THE NAV, WHATEVER + # THE TENANT DOCUMENT STILL CARRIES. + # + # W33-T44 retired `ut_odoo_customers` / `ut_odoo_products` in favour of `customer_data` / + # `product_data` — ONE SUBJECT, ONE DATABASE — and `apply_plan` pops them from a tenant's + # document on the next rebuild. The owner then reported *"Odoo products"* TWICE, and a live + # label census returned `{'Odoo products': 2}`: the retirement MECHANISM was correct and the + # PATH DID NOT RUN, for causes booked separately (D-251's `frozen(rt)` early return, and the + # rebuild being scoped to `is_royal`). + # + # ⛔ SO THIS IS A READ-SIDE GUARD, NOT THE MIGRATION, AND THE DIFFERENCE IS STATED RATHER THAN + # BLURRED: the stale ROW may still sit in a tenant's document, and removing it is + # `apply_plan`'s job in `odoo_relational.py`. What this line guarantees is that a document + # which still carries one cannot put a SECOND "Odoo products" on somebody's rail while that + # migration has not reached them. A defect whose fix depends on a migration running is a + # defect that comes back on the one tenant the migration missed + # [[a-migration-that-runs-on-the-next-write]]. + # + # ⚠ `RETIRED_KEYS` is IMPORTED, never re-typed here. A hand-copied tuple is exactly the drift + # this guard exists to prevent, and it would go stale the first time a third key is retired. + # ⚠ Inside the same `try:` as the rest of the user-table read on purpose — if the import + # fails, the block's own `except` marks the nav DEGRADED, which is the honest outcome. A + # guard that silently stops guarding is worse than one that says it could not run. + try: + from odoo_relational import RETIRED_KEYS as _retired + except Exception: + _retired = () + for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin, + st=_lent): + if e["key"] in _retired: + continue + if not _perm_scope.nav_may_open(session.user, e["key"], st=_lent): + continue + pages.append({"key": e["key"], "label": e["label"], + "source": e.get("source") or "Blank", "chrome": "main"}) + except Exception: + # ⭐⭐ W31-T11 (owner item 6b) — STILL SWALLOWED, NO LONGER SILENT. + # + # A store blip must not take the whole nav down with it, and that half stands. What + # changed is that it used to answer **200 OK with every database missing** and say + # nothing — so a client cannot tell "this tenant has no databases" from "we could not + # read them", and the rail renders the same complete-looking thing either way. That is + # the owner's *"Connectors and Automation module still disappears"* class of report: + # the payload is a claim about what exists, and a claim it could not verify has to be + # marked as such. `degraded` is that mark; the shell renders it in the affected slot. + _degraded.append("databases") + pass + if not pages: + if tmods != "all": + # A provisioned tenant with no modules and no databases YET is a legitimate empty + # state, not a misconfigured account — the client renders "create your first + # database", and X2's never-an-empty-200 rule is honoured by saying WHY it is + # empty rather than leaving 200-[] ambiguous. + return {"pages": [], "landing": None, "empty": "no_databases"} + raise err(403, "no_surfaces", + "your account has no dashboards assigned. Ask an administrator.") + # WAVE 19 (R8 / C1): the tenant's name + icon overrides, merged LAST — after the registry + # rows, after the tenant module filter, after the user tables. Merged HERE rather than + # applied by the client for one reason: `label` is what every reader of this payload shows, + # including the Settings modal's `moduleLabels` map, so a client-side merge would put the + # override in one door and the registry label in the other. + # + # ⛔ THIS MUTATES `pages` IN PLACE, WHICH IS SAFE ONLY BECAUSE `perms.nav_pages` BUILDS A + # FRESH `row = {...}` PER CALL (perms.py:194) and the ut_ rows are built fresh here. If + # either ever starts handing back cached or module-level dicts, this loop would write one + # tenant's chosen label into the object the NEXT tenant's request reads — a cross-tenant + # leak with no symptom until two tenants rename the same registry key. Copy the rows before + # merging on the day that invariant changes. + meta = _read_nav_meta(session.runtime) + # ⛔⛔ W35-T43 (rulings R4/R7) — THE MARK-IMPORTANT COUNTS ARE NO LONGER COMPUTED HERE. + # + # W34-T10 put them on this payload with a 2.5 s budget and a `degraded` entry, spent in RECENTS + # ORDER so a cold container would reach the databases somebody actually works in first. Every + # one of those was a correct answer to the wrong question: the block was `N` per-database store + # reads on the route D-175 spent a wave reducing to ONE, and no ordering makes an N-read block + # free. MEASURED on a 13-database fixture: **14 view-bucket reads and 14 `important` stamps per + # `/nav`** before this ticket, **0 and 0** after. + # + # R7 moves the question to `GET /starred/counts`, called AFTER the page paints. A slow answer + # there is a late badge; a slow answer here was a late RAIL. Closes D-288 and D-289. + # ⚠ `recents` STAYS and is still ONE read — it is the Home landing's own payload (C10/R7 of wave + # 23), and it was only computed this early so the deleted budget could spend itself in its order. + _page_keys = {str(p.get("key", "")) for p in pages} + recents = _read_recents(session.runtime, session.uname, _page_keys) + # Wave 21 (C3): the definitions, once — `manage`/`canDelete` below answer from `createdBy`. + # WAVE 27 (C9): `locked` answers from `recordMode`, off the same one read. + # ⭐ W31-T10: that read is now the SAME one the merge above did — it used to be a second, + # independent `all_tables`, which is why D-175 called this route `2 + N` rather than `1 + N`. + # ⚠ The fail-closed defaults still hold: both are initialised before the try above, so an + # import or store failure leaves `_ut_locked_mode` empty and nothing is marked locked. + for p in pages: + # `manage` (R14): may THIS session change this row's icon/name? Answered HERE because + # the server is the only end that knows — the client cannot see who created a user + # table. Additive and FAIL-CLOSED (absent reads as "no"), so the rail offers a control + # only where the write would actually land, and the route re-checks it regardless. + # + # ⛔ WAVE 21 (C3/W-5): the "presence IS the answer" shortcut DIED in wave 20 — the + # de5037f share-grant admission widened `may_open`, so a row's presence now includes + # databases merely SHARED to this viewer. `manage` (rename/icon) and `canDelete` are + # therefore answered from the DEFINITION: creator-or-admin strictly, matching the + # walls the PATCH and DELETE routes actually enforce. A grantee sees the row and no + # controls — the honest shape (the old code offered rename to users the route 403'd). + key = str(p.get("key", "")) + if key.startswith("ut_"): + _creator = str((_ut_defs.get(key) or {}).get("createdBy") or "") + _mine = bool(session.admin or (_creator and _creator == session.uname)) + p["manage"] = _mine + p["canDelete"] = _mine + # ⭐ WAVE 27 item 3 (contract C9) — A LOCKED DATABASE SAYS SO IN THE RAIL. + # + # "Locked" is the owner's item-4 vocabulary and it means exactly ONE thing (DESIGN.md + # §4, THE THREE LOCKS): RECORDS cannot be added, deleted or edited — **fields still + # can**. The automation-owned IG child datasets (posts, comments, snapshots) are the + # live example; their rows arrive from the engine, so a "+" row there could only + # refuse, which R8 calls a fake affordance. + # + # ⚠ THE AUTHORITY IS `user_tables.records_mutable`, NOT THIS LINE. The comparison is + # inlined only because `_ut_defs` is already in hand — calling the predicate per row + # would be one store read per database on every nav request — and the VALUE comes + # from the module's own constant rather than a copied string, so the two cannot drift + # to different answers. If that predicate ever grows a second condition, this must + # become a call. + # + # ⚠ ABSENT READS AS UNLOCKED, and that is the safe direction here even though it is + # the opposite of `manage`'s fail-closed: the lock ICON is an affordance hint, while + # the actual refusal is `routes_tables._records_or_refuse`'s 403. A store blip costs + # a missing hint, never a write that should not have landed. + if (_ut_locked_mode + and (_ut_defs.get(key) or {}).get("recordMode") == _ut_locked_mode): + p["locked"] = True + elif session.admin: + p["manage"] = True + # ⛔ W35-T43 — `important` IS NO LONGER STAMPED HERE. W34-T10 emitted it for every database + # this route could read, including `{marked: 0, counted: 0, partial: false}`, so a consumer + # never had to test for the key. R4 retires the count from the rail and the flyout entirely + # (A's W35-T07 is the client half) and R7 moves the numbers to `GET /starred/counts`. + # ⚠ `nav.ts::NavPage` still DECLARES `important` on the client until A's ticket lands; an + # absent key reads as `undefined` there, which is the same thing the optional field already + # meant for a non-database row. Flagged to A rather than assumed harmless. + entry = meta.get(p.get("key")) if meta else None + if not entry: + continue + if entry.get("icon"): + p["icon"] = entry["icon"] + if entry.get("name"): + p["label"] = entry["name"] + landing = perms.landing_page(session.user) + if landing and not any(p.get("key") == landing for p in pages): + landing = pages[0].get("key") + # WAVE 23 (C10 / R7) — the Home landing's recents, on the payload the client already asks + # for. A second round trip for a list this short, computed from a bucket this route is + # already holding the store open for, would be a request per page load for no gain. + # + # ⛔ `allowed` IS THE ASSEMBLED PAGE LIST — the rows this route just built, `ut_*` databases + # included. Pruning against `perms.nav_pages` alone would silently drop every user database + # from Home's recents: the exact surface R7 is about, invisible, with every gate green. + # + # ⚠ WAVE 27 (item 6): the OTHER consumer of that narrower set — the folder-placement door — + # had the identical bug and nobody connected the two for four waves. It is fixed at the + # source now (`_placeable_top_keys`, which merges the same `may_open`-filtered listing), so + # both doors finally agree on what a placeable key is. This block keeps using the assembled + # list because it already holds it: re-enumerating here would be a second store read for an + # answer sitting in a local variable. + # + # ⚠ W34-T10 MOVED THE READ, NOT THE RULE. `recents` is now computed ABOVE the enrichment loop, + # because the important-count block spends its budget in RECENTS ORDER (see there). It is still + # ONE read of `nav_recents`, still pruned against the assembled page list, and it is used here + # unchanged — a second `_read_recents` call would be the extra store read this comment forbids. + # ⭐ W31-T11 — TWO KINDS OF ABSENCE, NAMED SEPARATELY, and both keys are ALWAYS PRESENT. + # `omitted` — this workspace's catalogue does not include these modules. Deliberate. + # `degraded` — a part of this payload could not be read. NOT deliberate, and the shell says + # so in the affected slot instead of rendering a confident nothing. + # ⚠ A key a consumer has to test for is a key a consumer forgets to test for; both ship as + # `[]` rather than being omitted when empty, which is the same rule `limits` follows. + return {"pages": pages, "landing": landing, "recents": recents, + "omitted": _omitted, "degraded": _degraded} + + +@router.post("/nav/opened") +def nav_opened(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """WAVE 23 (C10) — stamp a page as JUST OPENED. Fire-and-forget from the client. + + The client calls this on every route commit, so this is the only write in this file on a + hot path, and three things follow from that: + + · `flush='async'` — the coalescing mode ([[store-async-flush]]). A blocking upload per + page open against an HF-Dataset-backed store would put a network round trip inside every + navigation. `nav_prefs`/`nav_meta` stay `sync` because a folder rename is not a hot path; + this is. + · NO VALIDATION OF THE KEY against the nav. Building the page list to check one string + would make the stamp cost more than the navigation that triggered it — and it would buy + nothing, because the READ prunes to what the session may currently see. An unknown or + revoked key is stored and never served back. + · THE MAP IS CAPPED HERE TOO. Read-side capping alone would let a hostile or buggy client + grow one user's document without bound; `_MAX_RECENTS` entries survive, oldest first to + go, which is the same rule the read applies. + """ + body = body if isinstance(body, dict) else {} + key = str(body.get("key") or "").strip()[:60] + if not key: + raise err(400, "bad_request", "no page was named") + if not session.runtime.available(): + raise err(503, "store_unavailable", + "the tenant store is unavailable. Nothing was recorded.") + stamp, uname = _now(), session.uname + + def _up(data): + data = data if isinstance(data, dict) else {} + mine = dict(data.get(uname) or {}) if isinstance(data.get(uname), dict) else {} + mine[key] = stamp + if len(mine) > _MAX_RECENTS: + # Oldest first. `int(v)` guarded: a stamp an older build wrote in another shape + # sorts as 0 and is the first thing evicted, which is the right answer for a value + # this route can no longer read. + def _at(item): + try: + return int(item[1]) + except (TypeError, ValueError): + return 0 + mine = dict(sorted(mine.items(), key=_at, reverse=True)[:_MAX_RECENTS]) + data[uname] = mine + return data + + try: + session.runtime.update(_NAV_RECENTS_KEY, _up, flush='async') + except Exception: + raise err(503, "store_unavailable", + "the tenant store refused the write. Nothing was recorded.") + return {"key": key, "at": stamp} + + +@router.post("/nav/meta") +def save_nav_meta(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """WAVE 19 (R8 / C1) — set one database's icon and/or name, tenant-wide. + + A PATCH OF ONE KEY, not the wholesale replace `/nav/prefs` uses two routes up, and the + asymmetry is deliberate. Prefs are one user's complete picture of their own rail, so + replacing the document whole is what makes a deleted folder stay deleted. This bucket is + shared by every admin in the tenant: a wholesale write here means whoever saves last + silently erases what the other one named while their tab was open. + + THREE WALLS, all fail-closed: + · the SESSION must be able to open the key at all (the same predicate the nav uses, so + the rail and this route cannot disagree about what exists); + · the WRITE is admin-only — renaming a database is a change every user in the tenant + sees, which is the definition of an administrative act here; + · `name` is refused outright for a non-`ut_` key. Built-in labels are compiled registry + literals: honouring an override would leave this payload and `core/registry.py` + calling the same module two different things, and the wave-16 lesson is that the + client must not be the place that decides what a payload meant. + + `icon: null` CLEARS. An absent field is untouched — which is what makes a rename and an + icon change two independent writes rather than a race between them. + """ + body = body if isinstance(body, dict) else {} + key = str(body.get("key") or "").strip() + if not key: + raise err(400, "bad_request", "no database was named") + # ── THE WALL (ruling R14, 2026-08-04) ──────────────────────────────────────────────────── + # TWO DOORS, because a `ut_` database and a built-in module are owned by different people. + # + # · `ut_*` — the table's CREATOR or a tenant admin, which is exactly what + # `user_tables.may_open` already means. A database you made is yours to name; requiring + # an admin for that was the C1 consequence B flagged and R14 resolved. `session.require` + # is deliberately NOT used here: it is the MODULE gate and would 403 every ut_ key, + # because a user table is not a module. Same split `/nav/schema/{key}` makes. + # · everything else — admin only, and icon only. There is no owner of `customer_data` to + # defer to, and its label is a compiled registry literal (refused below regardless). + if key.startswith("ut_"): + import core.user_tables as user_tables + if not user_tables.get(key, st=session.runtime) or not user_tables.may_open( + key, session.uname, session.admin, st=session.runtime): + raise err(403, "forbidden", "that database belongs to another user") + else: + if not session.admin: + raise err(403, "forbidden", + "only an administrator can change a built-in database's icon") + session.require(key) + + patch = {} + if "icon" in body: + icon = _clean_icon(body.get("icon")) + if body.get("icon") is not None and icon is None: + # Loud, not silent. A shape this build does not know is a CLIENT that has drifted + # from this whitelist, and answering 200 to a write that stored nothing is how + # that drift stays invisible until a user reports "my icon keeps resetting". + raise err(400, "bad_icon", "that icon is not one of the available shapes and tones") + patch["icon"] = icon + if "name" in body: + if not key.startswith("ut_"): + raise err(400, "name_not_allowed", + "only a database you created can be renamed. This one's name comes " + "from the module registry") + name = " ".join(str(body.get("name") or "").split())[:_MAX_NAV_NAME] + if not name: + raise err(400, "bad_request", "a database needs a name") + patch["name"] = name + if not patch: + raise err(400, "bad_request", "nothing to change") + + if not session.runtime.available(): + raise err(503, "store_unavailable", + "the tenant store is unavailable. Nothing was saved.") + + def _up(data): + data = data if isinstance(data, dict) else {} + entry = dict(data.get(key) or {}) if isinstance(data.get(key), dict) else {} + for field, value in patch.items(): + if value is None: + entry.pop(field, None) # an explicit null CLEARS + else: + entry[field] = value + # An entry with nothing left in it is removed rather than stored empty: the read side + # skips empties anyway, and a bucket that accumulates `{}` per key is a document that + # grows forever and says nothing. + if entry: + data[key] = entry + else: + data.pop(key, None) + return data + + try: + session.runtime.update(_NAV_META_KEY, _up) + except Exception: + raise err(503, "store_unavailable", + "the change was not saved: the store refused the write.") + return {"key": key, "meta": _read_nav_meta(session.runtime).get(key, {})} + + +@router.get("/nav/prefs") +def nav_prefs(session: Session = Depends(require_session)): + """This user's database-list folders, re-validated at serve time against what they may + currently see (a revoked page's placement vanishes with the page, and returns with it).""" + try: + stored = (session.runtime.get(_NAV_PREFS_KEY) or {}).get(session.uname) or {} + except Exception: + stored = {} + keys, enumerated = _placeable_top_keys(session) + return {"prefs": _clean_nav_prefs(stored, keys, keep_unknown_ut=not enumerated)} + + +@router.post("/nav/prefs") +def save_nav_prefs(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Wholesale replace, like the table folder stratum — the client sent its complete + picture, validated here; a partial merge would resurrect deleted folders forever.""" + keys, enumerated = _placeable_top_keys(session) + clean = _clean_nav_prefs(body or {}, keys, keep_unknown_ut=not enumerated) + if not session.runtime.available(): + raise err(503, "store_unavailable", + "the tenant store is unavailable. Nothing was saved.") + + def _up(data): + data = data if isinstance(data, dict) else {} + if clean["folders"] or clean["placement"]: + data[session.uname] = clean + else: + data.pop(session.uname, None) + return data + + try: + session.runtime.update(_NAV_PREFS_KEY, _up) + except Exception: + raise err(503, "store_unavailable", + "the folder change was not saved: the store refused the write.") + return {"prefs": clean} + + +@router.get("/nav/schema/{key}") +def nav_schema(key: str, session: Session = Depends(require_session)): + """The database's schema drawer payload: its field contract + the semantic measures this + session may build with. + + TWO WALLS, ANSWERING TWO QUESTIONS, AND THEY FAIL DIFFERENTLY ON PURPOSE. The DATABASE wall + is the same predicate the nav uses — a key this session may not open answers 403, never a + redacted schema. The FIELD wall runs *after* it, on a database this session may open, and + its whole job is to narrow: the drawer serves the columns this reader receives everywhere + else, and no more. + + ⭐⭐ W38-T18 — THE FIELD WALL IS NEW HERE AND THIS DOOR IS WHERE IT WAS MISSING. Every grid + door has stripped hidden columns from both wires since W36-T21; the schema drawer beside them + served the raw contract to anyone `require`/`may_open` admitted. Owner instruction 10 — + *"Metrics field lookback and value must abide by the Filter permissioning"* — is satisfied on + the picker side already (the condition kit takes `fields` as a PROP off a payload that is + walled upstream), so this route is the one place the same reader could still learn the name, + the label and the VALUE VOCABULARY of a column their grid refuses them. + """ + # Wave 18 C3-UT: a user table's schema is its own definition, walled by ITS predicate + # (`may_open`) rather than the module grant machinery — `session.require` would 403 every + # ut key because a user table is deliberately not a module. + if key.startswith("ut_"): + import core.user_tables as user_tables + defn = user_tables.get(key, st=session.runtime) + if not defn or not user_tables.may_open(key, session.uname, session.admin, + st=session.runtime): + raise err(403, "forbidden", "that database belongs to another user") + # ⭐⭐ W38-T18 — THE FIELD WALL ON A `ut_*` DATABASE, WHICH THIS DOOR HAS NEVER RUN. + # `may_open` answers *IF* you reach the database and says nothing about WHICH COLUMNS, so + # this drawer handed `defn["fields"]` whole to every account it admitted. The bypass is + # older than this ticket (it dates to the wall itself); what changed is the CATEGORY it + # leaks — since W38-T16 a hidden `ut_*` column means an administrator's `hiddenFields` + # *or* a column granted away, and a share door whose schema drawer still names the column + # tells the ungranted reader exactly what they were refused. + # ⚠ MERGED BEFORE THE WALL, NEVER AFTER — the same order `routes_tables.ut_assembly` + # runs, and for the reason `_ut_shared_fields` gives one door over: the closure must see + # the WHOLE contract, or a definition field whose formula reads a hidden tenant-wide + # column sits outside its reach and carries that value out wearing a second name. + # ⛔ ONLY THE DEFINITION'S OWN FIELDS ARE RETURNED. This drawer has never published the + # shared stratum and this ticket is not where it starts; widening the list handed to the + # closure can only ever ADD to the hidden set, never remove from it. + import routes_tables as _rt + base = list(defn.get("fields") or []) + hidden = _rt._ut_hidden(session, key, + _rt._ut_shared_fields(session, key, base), + st=session.runtime) + # WAVE 19 (R8) — the drawer wears the RENAMED name. A rail that says one thing and a + # schema panel opened from it that says another is the drift a rename is supposed to + # remove, not create. + return {"key": key, + "label": (_read_nav_meta(session.runtime).get(key, {}).get("name") + or defn.get("label") or key), + "source": defn.get("source") or "Blank", + "fields": [{"key": f["key"], "label": f["label"], "type": f["type"], + "source": f.get("source") or "overlay", + "description": str(f.get("description") or "")} + for f in base if f.get("key") not in hidden], + "measures": []} + session.require(key) + pages = perms.nav_pages(session.user) or [] + page = next((p for p in pages if p.get("key") == key), None) + if page is None: + raise err(404, "unknown_page", f"{key!r} is not a database this session can see") + fields = [] + if key in ("customer_data", "cohort", "customers"): + try: + import aios_grid + import routes_customers as _rc + # ⭐⭐ W38-T18 — THE FIELD WALL ON THE CUSTOMER CONTRACT, AND THE `options` LINE FOUR + # ROWS DOWN IS WHY IT IS A LEAK RATHER THAN AN UNTIDINESS. A schema drawer that names + # a hidden column has told the reader it exists; one that ships `options` has handed + # them the VALUE VOCABULARY of every choice and status column their grid refuses — + # the set of statuses the business uses, read straight off a door with no wall. + # ⛔ `_hidden_for` IS REUSED, NOT RE-DERIVED, and the difference is not cosmetic. + # `hidden_keys(user, MODULE, aios_grid.FIELDS)` is the obvious spelling and it is + # INCOMPLETE: that function's own docstring records that a wall computed from the + # canonical list alone answers "not hidden" for every runtime column, so the + # tenant-wide stratum never reaches the closure and a formula over a hidden column + # comes back. One evaluator, one caller, no second idea of what this session hides. + # ⚠ INSIDE THE `try`, DELIBERATELY. If the wall cannot be computed the route already + # answers with an empty contract rather than a full one — fail-closed is the only + # direction a schema door may fail, and the gate's ADMIN control is what stops that + # silent empty being mistaken for a wall that worked. + hidden = _rc._hidden_for(session) + for f in aios_grid.FIELDS: + if f["key"] in hidden: + continue + entry = {"key": f["key"], "label": f["label"], "type": f["type"], + "source": f["source"], + "description": str(f.get("description") or "")} + if f.get("options"): + entry["options"] = list(f["options"]) + fields.append(entry) + except Exception: + fields = [] + measures = [] + try: + from core import measure_resolve + import core.perm_scope as _perm_scope + team_id = perms.scope_team_id(session.user) + # ⭐⭐ W38-T19 — THE METRICS CAPABILITY REACHES THIS DRAWER TOO, AND IT IS THE ONE DOOR + # THE THREE GRID ASSEMBLIES DO NOT SPEAK FOR. This route calls `measure_resolve.offer` + # ITSELF rather than reading a `measures` key off an assembly, so a wall applied in + # `routes_customers` / `routes_products` / `routes_tables` narrows every grid and leaves + # the panel beside them naming the full lookback catalogue — the same one-door-short + # shape W38-T18 fixed for FIELDS in this very function, one key over. + # ⚠ THE UT_ BRANCH ABOVE RETURNS BEFORE THIS AND ALREADY SERVES `[]`, so there is + # nothing to narrow there; if it ever starts publishing an offer, it needs this line. + _offer = ((measure_resolve.offer(team_id) or []) + if _perm_scope.may_metrics(session.user, key) else []) + for m in _offer: + measures.append({"key": str(m.get("key") or ""), + "label": str(m.get("label") or m.get("key") or ""), + "type": str(m.get("type") or "")}) + except Exception: + measures = [] + out = {"key": key, "label": page.get("label") or key, + "source": page.get("source") or "", "fields": fields, "measures": measures} + if not fields: + # Honest, never a mock: a database whose contract is not yet published says so. + out["note"] = "This database has not published a field contract yet." + return out diff --git a/api/routes_oauth.py b/api/routes_oauth.py index 7795caf496a8598b3c52b4cf5b0663939012d5c8..8e49c41134aafde8a9738e53d63775c2d76fea88 100644 --- a/api/routes_oauth.py +++ b/api/routes_oauth.py @@ -1,114 +1,114 @@ -"""routes_oauth.py — the OAuth connector surface (wave 22, contract C5 + A2/A3 / R12). - -Thin over `oauth_connect`, the way `routes_automation` is thin over the engine: sessions, -shapes and status codes here; every decision that could be wrong lives in the module a gate -can drive without a server. GENERIC over `{provider}` (C5-A2): the routes read the registry, -so the day a second provider lands here is the day nothing in this file changes. - -MOUNTED FROM `routes_automation` (not `main.py`): this wave's ownership fence gives no session -`main.py`, and `routes_automation` is already included there — so this router rides inside it -(`/api/v1` + `/oauth/...`). Lifting the include into `main.py` later is a two-line change that -alters no path. - -⚠ THE TWO REDIRECT LAWS (A3): `/{provider}/start` answers **302 to the provider's consent -screen** — it is a top-level navigation the client reaches by ``, never JSON. The -callback 302s BACK to the return path the `state` carried (relative-only, sanitised by -`oauth_connect.safe_next`), so the user lands where they left — connected or not, whatever -went wrong rides in the query string; a dead-end error page where the app used to be reads as -"the product broke", not "the connect failed". -""" -import os - -from fastapi import APIRouter, Depends, Request -from fastapi.responses import RedirectResponse - -import oauth_connect -from deps import Session, err, require_session - -router = APIRouter(prefix="/oauth") - - -def _redirect_uri(request: Request, provider: str) -> str: - """The redirect URI this deployment registers at the provider — env-pinned when the - container sits behind a proxy that rewrites the scheme (the HF Space), else derived from - the request. MUST match a console-registered URI verbatim, so it is computed in exactly - one place. - - ⭐ WAVE 29 (R4): `deploy_web.py` now PUSHES `AIOS_PUBLIC_BASE` on every deploy, defaulted to - the same URL as `APP_BASE_URL`, so the pinned branch is the one that runs in production and - the request-derived fallback below is effectively dev-only. - ⛔ THAT MAKES THIS FUNCTION A CUSTOM-DOMAIN COUPLING, not merely a scheme fix. Whatever host - this returns is where the provider sends the user BACK, and the session cookie is host-only - (`aios_session.py:114-117`, no `domain=`) — so a callback base that disagrees with the host - the user actually browsed plants the session on the wrong hostname and they return logged - out. Moving the app to a new hostname means moving this value AND re-registering the - resulting URI in the provider console; one without the other fails closed. - Runbook: `.claude/wiki/research/loopable-domain-runbook.md`.""" - base = (os.environ.get("AIOS_PUBLIC_BASE") or "").strip().rstrip("/") - if not base: - base = f"{request.url.scheme}://{request.url.netloc}" - return f"{base}/api/v1/oauth/{provider}/callback" - - -@router.get("/status") -def oauth_status(session: Session = Depends(require_session)): - """C5's status shape for the SESSION user, one entry per registry provider: - `{google: {connected, email, reconnect, configured}}` today. The bit the email trigger's - `ready` reads through.""" - return oauth_connect.status(session.runtime, session.uname) - - -def _offered_or_404(provider: str): - """⛔⛔ W32-T14 / OWNER ITEM 12 / R6 — A PROVIDER THE PRODUCT DOES NOT OFFER HAS NO DOOR. - - The owner pasted the failure this replaces: clicking Connect on the Google card answered - `503 oauth_unavailable` as raw JSON. R6's fix is not a nicer error — it is that the flow is - not offered at all, so the honest status is the one for a URL that does not exist. `404` - rather than `503`, deliberately: a 503 says *"come back later"* about a door that is not - coming back until somebody pays for CASA verification (D-45, ~$540–1,800/yr). - - ⚠ Every door in this router goes through it, START included, because the JSON the owner saw - came from the start route and a guard on one leg is a guard on one leg. - """ - if not oauth_connect.offered(provider): - raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider") - - -@router.get("/{provider}/start") -def oauth_start(provider: str, request: Request, next: str = "", - session: Session = Depends(require_session)): - """302 to the provider's consent screen (A3 — a navigation, never JSON). `?next=` is the - RELATIVE path the callback returns the browser to; it rides inside the single-use state, - sanitised, so the round trip cannot be steered off-origin.""" - _offered_or_404(provider) - url, problem = oauth_connect.start(provider, session.uname, - _redirect_uri(request, provider), next_path=next) - if problem: - raise err(503 if "not configured" in problem else 404, "oauth_unavailable", problem) - return RedirectResponse(url, status_code=302) - - -@router.get("/{provider}/callback") -def oauth_callback(provider: str, request: Request, - session: Session = Depends(require_session), - state: str = "", code: str = "", error: str = ""): - """The provider's redirect target. Exchanges the code, stores the per-user slot, and sends - the browser back to the state's return path — connected or not (see module header).""" - _offered_or_404(provider) - if error: - home = "/#/" - return RedirectResponse(f"{home}?oauthError={error[:80]}", status_code=302) - email, home, problem = oauth_connect.callback(session.runtime, session.uname, state, code) - sep = "&" if "?" in home else "?" - if problem: - return RedirectResponse(f"{home}{sep}oauthError=connect_failed", status_code=302) - return RedirectResponse(f"{home}{sep}connected={provider}", status_code=302) - - -@router.post("/{provider}/disconnect") -def oauth_disconnect(provider: str, session: Session = Depends(require_session)): - _offered_or_404(provider) - if oauth_connect.provider_def(provider) is None: - raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider") - oauth_connect.disconnect(session.runtime, session.uname, provider) - return {"disconnected": provider} +"""routes_oauth.py — the OAuth connector surface (wave 22, contract C5 + A2/A3 / R12). + +Thin over `oauth_connect`, the way `routes_automation` is thin over the engine: sessions, +shapes and status codes here; every decision that could be wrong lives in the module a gate +can drive without a server. GENERIC over `{provider}` (C5-A2): the routes read the registry, +so the day a second provider lands here is the day nothing in this file changes. + +MOUNTED FROM `routes_automation` (not `main.py`): this wave's ownership fence gives no session +`main.py`, and `routes_automation` is already included there — so this router rides inside it +(`/api/v1` + `/oauth/...`). Lifting the include into `main.py` later is a two-line change that +alters no path. + +⚠ THE TWO REDIRECT LAWS (A3): `/{provider}/start` answers **302 to the provider's consent +screen** — it is a top-level navigation the client reaches by ``, never JSON. The +callback 302s BACK to the return path the `state` carried (relative-only, sanitised by +`oauth_connect.safe_next`), so the user lands where they left — connected or not, whatever +went wrong rides in the query string; a dead-end error page where the app used to be reads as +"the product broke", not "the connect failed". +""" +import os + +from fastapi import APIRouter, Depends, Request +from fastapi.responses import RedirectResponse + +import oauth_connect +from deps import Session, err, require_session + +router = APIRouter(prefix="/oauth") + + +def _redirect_uri(request: Request, provider: str) -> str: + """The redirect URI this deployment registers at the provider — env-pinned when the + container sits behind a proxy that rewrites the scheme (the HF Space), else derived from + the request. MUST match a console-registered URI verbatim, so it is computed in exactly + one place. + + ⭐ WAVE 29 (R4): `deploy_web.py` now PUSHES `AIOS_PUBLIC_BASE` on every deploy, defaulted to + the same URL as `APP_BASE_URL`, so the pinned branch is the one that runs in production and + the request-derived fallback below is effectively dev-only. + ⛔ THAT MAKES THIS FUNCTION A CUSTOM-DOMAIN COUPLING, not merely a scheme fix. Whatever host + this returns is where the provider sends the user BACK, and the session cookie is host-only + (`aios_session.py:114-117`, no `domain=`) — so a callback base that disagrees with the host + the user actually browsed plants the session on the wrong hostname and they return logged + out. Moving the app to a new hostname means moving this value AND re-registering the + resulting URI in the provider console; one without the other fails closed. + Runbook: `.claude/wiki/research/loopable-domain-runbook.md`.""" + base = (os.environ.get("AIOS_PUBLIC_BASE") or "").strip().rstrip("/") + if not base: + base = f"{request.url.scheme}://{request.url.netloc}" + return f"{base}/api/v1/oauth/{provider}/callback" + + +@router.get("/status") +def oauth_status(session: Session = Depends(require_session)): + """C5's status shape for the SESSION user, one entry per registry provider: + `{google: {connected, email, reconnect, configured}}` today. The bit the email trigger's + `ready` reads through.""" + return oauth_connect.status(session.runtime, session.uname) + + +def _offered_or_404(provider: str): + """⛔⛔ W32-T14 / OWNER ITEM 12 / R6 — A PROVIDER THE PRODUCT DOES NOT OFFER HAS NO DOOR. + + The owner pasted the failure this replaces: clicking Connect on the Google card answered + `503 oauth_unavailable` as raw JSON. R6's fix is not a nicer error — it is that the flow is + not offered at all, so the honest status is the one for a URL that does not exist. `404` + rather than `503`, deliberately: a 503 says *"come back later"* about a door that is not + coming back until somebody pays for CASA verification (D-45, ~$540–1,800/yr). + + ⚠ Every door in this router goes through it, START included, because the JSON the owner saw + came from the start route and a guard on one leg is a guard on one leg. + """ + if not oauth_connect.offered(provider): + raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider") + + +@router.get("/{provider}/start") +def oauth_start(provider: str, request: Request, next: str = "", + session: Session = Depends(require_session)): + """302 to the provider's consent screen (A3 — a navigation, never JSON). `?next=` is the + RELATIVE path the callback returns the browser to; it rides inside the single-use state, + sanitised, so the round trip cannot be steered off-origin.""" + _offered_or_404(provider) + url, problem = oauth_connect.start(provider, session.uname, + _redirect_uri(request, provider), next_path=next) + if problem: + raise err(503 if "not configured" in problem else 404, "oauth_unavailable", problem) + return RedirectResponse(url, status_code=302) + + +@router.get("/{provider}/callback") +def oauth_callback(provider: str, request: Request, + session: Session = Depends(require_session), + state: str = "", code: str = "", error: str = ""): + """The provider's redirect target. Exchanges the code, stores the per-user slot, and sends + the browser back to the state's return path — connected or not (see module header).""" + _offered_or_404(provider) + if error: + home = "/#/" + return RedirectResponse(f"{home}?oauthError={error[:80]}", status_code=302) + email, home, problem = oauth_connect.callback(session.runtime, session.uname, state, code) + sep = "&" if "?" in home else "?" + if problem: + return RedirectResponse(f"{home}{sep}oauthError=connect_failed", status_code=302) + return RedirectResponse(f"{home}{sep}connected={provider}", status_code=302) + + +@router.post("/{provider}/disconnect") +def oauth_disconnect(provider: str, session: Session = Depends(require_session)): + _offered_or_404(provider) + if oauth_connect.provider_def(provider) is None: + raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider") + oauth_connect.disconnect(session.runtime, session.uname, provider) + return {"disconnected": provider} diff --git a/api/routes_publish.py b/api/routes_publish.py index 3c9d85cd01091f4cb99ec7874e1647bf54ba60bd..b0a8bbebf8820fd13b3bc96e8cf175a815430a99 100644 --- a/api/routes_publish.py +++ b/api/routes_publish.py @@ -1,917 +1,917 @@ -"""routes_publish.py — PUBLISH AN INTERFACE VIEW AS A LINK (wave 33, owner item 8b, ruling R5). - -The owner, verbatim: *"Have the ability to publish interface, when a user's view is an interface -(e.g. Map or Catalog), we should have the ability to publish a link that can be either accessed -publicly or with a password that the user that share it can toggle."* - -R5, in five clauses, and every one of them is load-bearing: - * publish = a per-view secret token in a **server-only bucket**; - * a `public | password` toggle the SHARER owns, with a **hashed** passphrase beside it; - * an **unauthenticated** read-only route plus a `#/v/` client route; - * the publish surface **projects only the columns the view shows** — never the whole table; - * **creator-or-admin** may publish, and **revoke ROTATES** the token. - -⛔ THIS FILE COPIES `routes_forms.py`'S CONSTRUCTION ON PURPOSE, AND THE TICKET SAID TO. That door -has been public since wave 23 and carries scar tissue no fresh design would reproduce: ONE refusal -for every resolution failure (so the route is not an oracle for which tokens are real), a -constant-time compare, a STREAMING body bound rather than `Body(...)` (FastAPI reads and parses the -whole body before the handler's first line, and `content-length` is caller-supplied), a sliding -rate window rather than a fixed bucket, and an index that is a POINTER, never a permission. - -⚠ WHAT IS DELIBERATELY *NOT* SHARED WITH `routes_forms.py`: the bucket. A form token buys a WRITE -door into a table; a publish token buys a READ projection of one view. Folding them into one index -would make a single leaked string ambiguous about which of the two it opens, and would make -`_resolve` return an object whose capability depends on a field rather than on which door was -knocked. Two buckets, two resolvers, one shape. - -⚠ AND NOT SHARED WITH SHARING. **Three systems now answer some version of "who can see this"** and -they are not the same question (audit S-4): the grant registry drives *Shared with me*, -`table_store.is_shared` GATES opening a view inside the app, and a published link is a THIRD thing -— an unauthenticated read of a projection, bound to a secret rather than to an account. A published -link is NOT a grant: it creates no registry row, appears in nobody's *Shared with me*, and cannot -be revoked by removing a person, because there is no person. - - python verify_forms.py # this file's gate; section 6 onward -""" -import hashlib -import hmac -import os -import secrets -import time - -from fastapi import APIRouter, Depends, Request - -from deps import Session, err, require_session - -router = APIRouter(prefix="/api/v1") - -#: The SERVER-ONLY index: `{token: {table, view, access, pw?, salt?, iter?, createdBy, createdAt}}`. -#: ⛔ IT IS NEVER `display.*`. Contract C1 puts two PRESENTATIONAL flags on the view spec -#: (`published`, `publishAccess`) precisely so the client has something to render, and the browser -#: writes that spec on every autosave — so anything secret living there would be echoed back to the -#: browser by construction. `aios_grid._clean_display`'s allowlist enforces the other half. -TOKENS_KEY = "publish_tokens" -TOKEN_BYTES = 24 - -#: Passphrase storage. PBKDF2-HMAC-SHA256 with a per-link salt. -#: ⚠ D-130 IS THE SCAR THIS AVOIDS: a form's "invited addresses" list is IDENTIFICATION — it says -#: who you claim to be and anyone may claim it. A passphrase is AUTHENTICATION. The difference is -#: not a stronger string, it is that the secret is never stored, never logged and never echoed. -PW_ITERATIONS = 240_000 -PW_SALT_BYTES = 16 -MIN_PASSPHRASE = 6 -MAX_PASSPHRASE = 128 - -#: v1 request protections, the same shape and the same numbers as the form door (contract C9), so -#: the two public routes cannot drift into different postures. ⚠ In-process, therefore PER WORKER. -RATE_WINDOW_S = 60 -RATE_PER_WINDOW = 30 -MAX_BODY_BYTES = 16 * 1024 - -_HITS: dict = {} - -#: The view kinds that may be published, i.e. R5's "interface". -#: ⚠ THE CLIENT'S SOURCE OF TRUTH IS `customer-grid/iconShapes.ts::MODE_GROUP` (wave 33 item 9 -#: moved `swipe` and `timeseries` into this group). This constant is the SERVER's copy and the two -#: are held in step by `verify_forms.py`, which parses `MODE_GROUP` and compares — because a -#: server list that silently drifts from the picker means a mode a user can create and cannot -#: publish, with nothing anywhere going red. A `grid` or `kanban` view is a re-shaping of a row -#: set; publishing one would be publishing the table, which is exactly what R5's projection clause -#: exists to prevent. -#: ⛔⛔ W33-T68 — `form` IS DELIBERATELY ABSENT, AND ITS ABSENCE IS THE FIX. -#: A `form` view's rows ARE the submissions people have sent it. Every other mode here re-shapes a -#: row set the publisher already curated; a form's row set is other people's answers, gathered under -#: an implicit promise that they go to the owner. Publishing one turned "share this interface" into -#: "serve the responses to anyone holding the link" — on the one unauthenticated door in the -#: product, with no wall between the link and the data. -#: ⚠ AND A FORM ALREADY HAS ITS OWN PUBLIC DOOR: `#/form/` through `routes_forms.py`, which -#: serves the BLANK form for submitting and never the stored rows. So this is not a capability -#: removed, it is a second door onto the same object that should never have existed beside the -#: first. Publishing a form to be filled in still works, at the URL that was always for it. -#: ⚠ `verify_forms.py` holds this list in step with the client's `MODE_GROUP`; the client must not -#: offer Publish on a form view, or the picker promises what this refuses. -PUBLISHABLE_MODES = ("map", "catalog", "swipe", "timeseries") - - -def _refuse(): - """THE ONE REFUSAL, for every way a published link can fail to resolve. - - Wrong token, revoked link, deleted view, deleted table, a view that stopped being an - interface, a wrong passphrase. ⛔ Callers must not branch a more specific message out of it: - the difference between "no such link" and "that link was revoked" tells an enumerator which - tokens are real, and the difference between "no such link" and "wrong passphrase" tells them - which links are worth guessing at. - """ - return err(403, "bad_publish_token", "that link is not valid") - - -def _same(a: str, b: str) -> bool: - """Constant-time. A `==` leaks a secret's prefix through timing, one character at a time.""" - return hmac.compare_digest(str(a or ""), str(b or "")) - - -def _client_ip(request: Request) -> str: - """⛔⛔ W33-T70 — `x-forwarded-for` IS GONE FROM THIS FUNCTION, AND THAT WAS THE WHOLE HOLE. - - The header is chosen by the caller. Keying a rate limit on it means an enumerator writes a new - value per request and every request lands in a fresh bucket: MEASURED at **100 of 100 admitted - through a 30-per-window ceiling**. A limiter with a caller-chosen key is not a limiter, and its - old docstring said the header was "a rate-limit key and NOTHING else" — which was exactly the - use it could not support. It is safe as a LOG field and as nothing else. - - The socket peer is the only thing here the caller cannot choose, so it is the key. - ⚠ AND BEHIND HF'S PROXY EVERY CALLER SHARES ONE PEER, which is why `_rate_ok` counts FAILURES - ONLY (see there). A per-peer ceiling over ALL traffic would be one global bucket, i.e. an alarm - that fires for everyone [[alarm-that-fires-for-everyone]] — the honest reading of a shared peer - is that we cannot separate callers, not that we should throttle them together. - """ - return request.client.host if request.client else "?" - - -def _rate_ok(key: str, now: float) -> bool: - """Is this caller UNDER the failure ceiling? Pure check — call `_note_failure` to count. - - A SLIDING window. A fixed bucket lets a caller spend a whole allowance at 11:59:59 and the - whole next one at 12:00:00 — i.e. double the limit, back to back, against a door whose entire - protection is that guessing a 24-byte token is slow. - - ⛔ W33-T70 — IT COUNTS FAILURES, NOT REQUESTS, and the split is what makes a shared proxy peer - survivable. A legitimate reader opens a link that RESOLVES, so they never touch the counter at - all; an enumerator produces nothing but misses. Counting every request under one shared peer - would have denied service to everybody the moment one guesser showed up — trading a token - oracle for an outage is not a fix. - """ - seen = [t for t in _HITS.get(key, ()) if now - t < RATE_WINDOW_S] - _HITS[key] = seen - return len(seen) < RATE_PER_WINDOW - - -def _note_failure(key: str, now: float) -> None: - """Record one failed resolution against this peer, and keep the table bounded.""" - seen = [t for t in _HITS.get(key, ()) if now - t < RATE_WINDOW_S] - seen.append(now) - _HITS[key] = seen - if len(_HITS) > 4096: - for k in [k for k, v in _HITS.items() if not v or now - v[-1] > RATE_WINDOW_S]: - _HITS.pop(k, None) - - -#: A salt used ONLY to burn the same PBKDF2 a real verification would, when there is no stored hash -#: to check against. Its value is irrelevant; its COST is the point. -_DUMMY_SALT = b"\x00" * 16 - - -def _equalise_pw_cost(entry) -> None: - """⛔⛔ W33-T70 — SPEND THE PBKDF2 EVEN WHEN THERE IS NOTHING TO VERIFY. - - MEASURED before this existed: a wrong TOKEN answered in ~0.4 ms and a wrong PASSPHRASE in - ~82 ms — a **206x gap with zero overlap across 24 samples**. So the refusal's careful wording - ("that link is not valid", identical for both) was undone by the clock: anyone could sort real - tokens from fake ones by timing alone, then spend their guesses only on the real ones. The - docstring on `_refuse` describes exactly the leak the code then handed over for free. - - Called on every path that fails BEFORE a passphrase check would have happened, so the cheap - branch costs what the expensive one costs. `iter` is taken from the entry when there is one, so - a link minted under a different iteration count stays indistinguishable too. - """ - iterations = PW_ITERATIONS - if isinstance(entry, dict): - try: - iterations = int(entry.get("iter") or PW_ITERATIONS) - except (TypeError, ValueError): - iterations = PW_ITERATIONS - _hash_pw("", _DUMMY_SALT, iterations) - - -def _public_base() -> str: - return (os.environ.get("AIOS_PUBLIC_BASE") or os.environ.get("APP_BASE_URL") or "").rstrip("/") - - -def _link_of(token: str) -> str: - return f"{_public_base()}/#/v/{token}" - - -def _tokens(rt) -> dict: - """This tenant's publish index, always a dict.""" - try: - found = rt.get(TOKENS_KEY) or {} - except Exception: # noqa: BLE001 - return {} - return found if isinstance(found, dict) else {} - - -def _hash_pw(passphrase: str, salt: bytes, iterations: int = PW_ITERATIONS) -> str: - return hashlib.pbkdf2_hmac("sha256", str(passphrase).encode("utf-8"), - salt, int(iterations)).hex() - - -def _pw_ok(entry: dict, passphrase: str) -> bool: - """Constant-time verify against the stored digest. - - ⛔ RETURNS FALSE, NEVER RAISES, and never distinguishes "this link has no passphrase stored" - from "the passphrase is wrong" — a `password` link whose hash went missing must refuse, not - fall open. [[default-must-pass-its-own-guard]] - """ - stored, salt_hex = str(entry.get("pw") or ""), str(entry.get("salt") or "") - if not stored or not salt_hex: - return False - try: - salt = bytes.fromhex(salt_hex) - except ValueError: - return False - got = _hash_pw(passphrase, salt, int(entry.get("iter") or PW_ITERATIONS)) - return _same(got, stored) - - -def _find_view(rt, table_key: str, view_id: str): - """`(view, owner_username)` from the table's workspace bucket, or `(None, "")`. - - The bucket shape is `{username: {views: {id: view}}}` — the same walk `routes_forms._find_view` - does, and the reason a view is addressable by id ALONE here: the id is unique within the table, - while the owner is what we are trying to discover. - """ - if not table_key or not view_id: - return None, "" - try: - bucket = rt.get(f"{table_key}_table_workspace") or {} - except Exception: # noqa: BLE001 - return None, "" - if not isinstance(bucket, dict): - return None, "" - for owner, blob in bucket.items(): - views = (blob or {}).get("views") if isinstance(blob, dict) else None - if isinstance(views, dict) and isinstance(views.get(view_id), dict): - return views[view_id], str(owner) - return None, "" - - -def _mode_of(view: dict) -> str: - """The view's display mode. Absent means `grid` — the stored default is "say nothing".""" - cfg = (view or {}).get("config") if isinstance(view, dict) else None - disp = (cfg or {}).get("display") if isinstance(cfg, dict) else None - return str((disp or {}).get("mode") or "grid") if isinstance(disp, dict) else "grid" - - -def _may_administer_view(session: Session, table_key: str, view_id: str): - """`(view, mode)` when this caller may publish THIS view, else raises. R5's creator-or-admin. - - ⛔ `session.uname` / `session.admin`, NOT `username` / `is_admin`. `Session` has no such - attributes, and reading one raises ABOVE this route's own `try:` — which is how D-107 arrived - as a bare plain-text 500 rather than as our JSON envelope. - """ - import core.table_store as table_store - - if not table_key.startswith("ut_"): - # The Odoo/registry grids are read-through mirrors with their own permission wall; a - # published projection of one would be a second, secret-gated door onto tenant data whose - # visibility the module gate is supposed to decide. - raise err(400, "not_a_database", "you can publish views on your own databases only") - # ⛔ A READ-THROUGH DATABASE CANNOT BE PUBLISHED, AND THE REFUSAL SAYS SO RATHER THAN MINTING - # A LINK THAT WILL NOT OPEN. `startswith("ut_")` admits `ut_odoo_*` and `ut_meta_*`, whose rows - # do not live in the tenant document at all — they are windowed out of the DuckDB mirror - # through a Session-bound path (`ut_assembly`, which 409s `window_required` on the big ones, - # D-174's lineage). An unauthenticated route has no session and therefore no such path, so a - # token minted here would resolve to a page that could never render. - # ⚠ THIS IS R6's SECOND SENTENCE, WHICH IS THE HALF THAT GETS DROPPED: a limit that genuinely - # cannot be removed must be REPORTED, with its cause, never silently enforced. Refusing at the - # MINT — where a person is standing in front of the answer — is the only place that reads as a - # sentence rather than as an empty page. - import core.user_tables as user_tables - - if user_tables.is_connected(table_key, st=session.runtime): - raise err(400, "connected_source", - "a database that reads through a connected source (Odoo, Meta Ads) cannot be " - "published as a public link: its rows are served from the tenant's mirror by a " - "signed-in request, and a public link has no session to serve them with") - view, owner = _find_view(session.runtime, table_key, view_id) - if not isinstance(view, dict): - raise err(404, "no_such_view", "that view no longer exists") - if not (owner == session.uname - or table_store._may_administer(view, session.uname, session.admin)): - raise err(403, "not_yours", "only this view's creator or an admin can publish it") - mode = _mode_of(view) - if mode not in PUBLISHABLE_MODES: - raise err(400, "not_an_interface", - "only an interface view (Map, Catalog, Swipe, Time-series, Form) can be " - "published as a link") - cfg = view.get("config") or {} - if cfg.get("cohortLock"): - raise err(400, "reader_scoped", - "this view is locked to a cohort, and a cohort's membership is resolved for " - "the person reading it — a public link has no reader, so the page would show " - "no rows at all. Publish a copy without the cohort lock.") - if _needs_a_reader(cfg.get("filters")): - raise err(400, "reader_scoped", - "this view filters on a cohort, a measure rule or a top-N slice, and each of " - "those is resolved for the person reading it — a public link has no reader, so " - "the page would show no rows at all. Publish a copy filtered on columns.") - return view, mode - - -#: How many rows a published page will serve. ⚠ R6 (no cap on connected-source data) does not -#: reach here twice over: a publishable database is `records_mutable`, i.e. the EDITABLE substrate -#: that keeps its bound (`user_tables.MAX_ROWS`), and a connected one is refused at the mint. What -#: R6's SECOND sentence does reach here is the reporting duty — a page that serves fewer rows than -#: the view has must SAY SO on the wire, with the cause, never just stop. -PUBLIC_ROW_CAP = 5000 - - -def _needs_a_reader(tree) -> bool: - """Does this filter tree contain a leaf that only a SIGNED-IN reader could resolve? - - ⛔ COHORTS, MEASURE RULES AND RANK SLICES ARE NOT PROPERTIES OF THE VIEW. Each is a SET the - host computes for the person asking — `filter_eval.EvalCtx`'s own docstring: *"each is an - answer a single ROW cannot compute … absent, the condition matches NOTHING rather than - everything"*. That default is right (it fails closed) and it is unusable here: an anonymous - page whose every row was silently filtered out is indistinguishable from a broken link, and - the reader has nobody to ask. Worse, `rank_sets` has NO server-side resolver anywhere in this - repo — `routes_alerts._evaluate` passes cohort/measure/today and nothing else — so a `topN` - view would serve zero rows on the server while showing twenty in the browser. - - ⭐ So such a view is refused AT THE MINT, where a person is standing in front of the answer. - That is R6's second sentence: a limit that genuinely cannot be removed is REPORTED with its - cause, never silently enforced. - - ⚠ THE PREDICATES ARE `filter_eval`'S OWN. Re-implementing "is this a cohort leaf?" here would - be a second evaluator that agrees today and drifts the day the leaf shape changes — and it - would drift SILENTLY, because the two answers only differ on views nobody has published yet. - [[one-evaluator-per-question]] - """ - from harness import filter_eval - - def walk(node) -> bool: - if isinstance(node, list): - return any(walk(n) for n in node) - if not isinstance(node, dict): - return False - if node.get("kind") == "group" or isinstance(node.get("children"), list): - return any(walk(n) for n in (node.get("children") or [])) - if filter_eval._is_cohort(node) or filter_eval._is_measure(node): - return True - return node.get("op") in filter_eval.RANK_OPS - - return walk(tree) - - -def _mirror_display(rt, table_key: str, view_id: str, published: bool, access: str = "password"): - """Keep contract C1's two PRESENTATIONAL flags on the stored view in step with this bucket. - - ⛔ WHY THIS EXISTS AT ALL, and it was found by a reviewer rather than by a gate: revoking a - link dropped the token and left `config.display.published: true` on the view, so the grid's - own UI would go on saying "published" about a link that no longer resolves. Two records of one - fact, and only one of them moving, is the [[flag-shipped-without-its-writer]] shape — here with - the writer present and the OTHER half forgotten. - - ⚠ THE BUCKET REMAINS THE TRUTH. These flags exist so the client has something to render - without asking; they are a MIRROR, never a source, and nothing in this module reads them back - to decide anything. `read_view_link` deliberately reports both so a drift is visible rather - than assumed away. - - ⚠ ONLY THE TWO LEGAL KEYS ARE WRITTEN, with E's fail-closed coercion reproduced exactly - (`aios_grid._clean_display`: `published: True` always carries a `publishAccess`, and anything - that is not the literal `public` stores as `password`) — so a value written here and a value - written by the browser cannot disagree. - """ - def _set(cur): - cur = dict(cur or {}) - for owner, blob in list(cur.items()): - views = (blob or {}).get("views") if isinstance(blob, dict) else None - view = views.get(view_id) if isinstance(views, dict) else None - if not isinstance(view, dict): - continue - cfg = dict(view.get("config") or {}) - disp = dict(cfg.get("display") or {}) - if not disp.get("mode"): - # No display block means no interface view; nothing here should invent one. - continue - if published: - disp["published"] = True - disp["publishAccess"] = "public" if access == "public" else "password" - else: - disp.pop("published", None) - disp.pop("publishAccess", None) - cfg["display"] = disp - cur[owner] = {**blob, "views": {**views, view_id: {**view, "config": cfg}}} - return cur - - try: - rt.update(f"{table_key}_table_workspace", _set, flush="sync") - except Exception: # noqa: BLE001 - # ⚠ SWALLOWED, and deliberately: the token bucket is the truth and it has already been - # written. A mirror that failed to update leaves the UI one refresh out of date, which is - # strictly better than a 500 on a publish that actually succeeded. - pass - - -def _entry_for(rt, table_key: str, view_id: str): - """`(token, entry)` for a view's existing link, or `(None, None)`.""" - for token, where in _tokens(rt).items(): - if (isinstance(where, dict) and where.get("table") == table_key - and where.get("view") == view_id): - return str(token), where - return None, None - - -def _state_of(token, entry) -> dict: - """The link's state as the SHARER may see it. Built key by key. - - ⛔ NO `**entry`. The stored blob carries `pw` and `salt`; a spread would put both on the wire - to the browser, which is the whole failure this file's bucket exists to avoid, and it would do - it silently the first time somebody added a field. - """ - if not token or not isinstance(entry, dict): - return {"published": False, "access": "password", "token": "", "url": ""} - return { - "published": True, - "access": "public" if entry.get("access") == "public" else "password", - "token": str(token), - "url": _link_of(str(token)), - # A boolean, never the digest and never the salt. - "hasPassphrase": bool(entry.get("pw")), - "createdBy": str(entry.get("createdBy") or ""), - } - - -def _resolve(token: str): - """`(runtime, tenant_slug, table_key, view, entry)` for a token, or None. - - ⛔ THE INDEX IS A POINTER, NEVER A PERMISSION. Holding a token means the sharer minted it for - THIS view; it does not mean the holder may read anything else, and nothing downstream of here - may widen the subject beyond the `(table, view)` pair the entry names. - - ⚠ IT WALKS EVERY TENANT, because an unauthenticated request carries no tenant. That is the - form door's shape too, and the reason `_same` is constant-time: the walk compares the caller's - string against every stored token in the deployment. - """ - from harness import runtime as _rt - - if not token or len(token) < 16: - return None - for slug in _rt.known_tenants(): - try: - rt = _rt.get_runtime(slug) - except Exception: # noqa: BLE001 - continue - for stored, where in _tokens(rt).items(): - if not _same(str(stored), token) or not isinstance(where, dict): - continue - table_key = str(where.get("table") or "") - view, _owner = _find_view(rt, table_key, str(where.get("view") or "")) - # A view deleted, or re-saved as a grid, since the link was minted. Both answer the - # ONE refusal — "that link is not valid" — rather than explaining which. - if not table_key or not isinstance(view, dict): - return None - if _mode_of(view) not in PUBLISHABLE_MODES: - return None - return rt, slug, table_key, view, where - return None - - -def _coord(value, limit: float): - """A real coordinate, or `None`. THE WALL that lets a map publish without publishing a column. - - ⛔ A VALUE TEST, NEVER A NAME TEST. A field merely CALLED `lat` proves nothing — a verifier put - the string `CANARY-LAT-AAA` in one and watched it reach the wire under the first version of - this code. Anything that is not a finite number inside the earth's range is not a location and - does not travel. - ⚠ `bool` is rejected explicitly: `isinstance(True, int)` is True in Python, and `float(True)` - is `1.0` — a checkbox column named `lat` would otherwise publish as a point off the coast of - Ghana. - """ - if value is None or isinstance(value, bool): - return None - try: - n = float(value) - except (TypeError, ValueError): - return None - return n if n == n and abs(n) <= limit else None - - -def _visible_keys(view: dict, fields: list) -> list: - """The columns this view SHOWS, in the view's own order. R5's projection clause, in one place. - - ⛔ `config.order` IS NOT THE ANSWER AND IS THE OBVIOUS WRONG ONE. `aios_grid._default_view_config` - builds `order` as `shown + hidden`, and `grid_events`' `view_upsert` APPENDS every remaining - field to it — so `order` is every column the table has, hidden ones included. `visible` is the - only allowlist that exists; there is no stored "hidden" key to subtract. - - ⚠ AND AN EMPTY `visible` IS NOT "NO COLUMNS". `CustomerGrid` falls back to the table's default - set when a saved view carries none, so a public page that read `[]` as an empty allowlist would - render blank — and one that read it as "all fields" would LEAK. The fallback is the same - predicate the default config uses (`field.default is not False`), taken from `aios_grid` rather - than restated here. - """ - import aios_grid - - by_key = {str(f.get("key")): f for f in fields if isinstance(f, dict)} - stored = [str(k) for k in (((view or {}).get("config") or {}).get("visible") or [])] - keep = [k for k in stored if k in by_key] - if keep: - return keep - # ⛔⛔ W33-T69 — A STALE `visible` MUST SERVE NOTHING, NOT THE TABLE DEFAULT. - # `delete_field` never prunes a view's stored `visible`, so a published view whose columns were - # later deleted and replaced arrives here with a NON-EMPTY `stored` of which nothing survives — - # and the fallback below then WIDENS the public payload to whatever the table declares by - # default. The publisher chose five columns; the anonymous reader gets the table's idea of - # sensible. That is a widening on the one unauthenticated door in the product. - # ⚠ THE DISTINCTION IS `stored` NON-EMPTY, NOT `keep` EMPTY. A view that never stored `visible` - # at all (a legacy publish, a view saved before the key existed) has no intent to honour and - # the default IS the right answer for it — that is what the fallback was written for. A view - # that stored five keys and has none left DID state an intent, and every column it named is - # gone: the honest answer is no columns, which renders as an empty published view rather than - # somebody else's data. - if stored: - return [] - try: - default_visible = (aios_grid._default_view_config(fields) or {}).get("visible") or [] - except Exception: # noqa: BLE001 - default_visible = [] - fallback = [str(k) for k in default_visible if str(k) in by_key] - return fallback or [str(f.get("key")) for f in fields if f.get("default") is not False] - - -def _public_view(rt, table_key: str, view: dict) -> dict: - """The ONLY bytes a valid token buys. Built KEY BY KEY — there is no `**row` in this function. - - ⛔ THE REASON THAT IS A RULE AND NOT A STYLE. `aios_grid.rows_from_pool` puts `pid`, `_created`, - `lat` and `lon` on EVERY row regardless of what the view shows, and the stored row dict carries - every column the table has. Serialising a row and deleting the fields we do not want inverts - the failure: a column added next wave is INCLUDED by default and nobody notices, whereas an - allowlist that has not learned about it merely omits it. `routes_forms._public_form` is the - shipped precedent and says the same thing about itself. - """ - import core.user_tables as user_tables - from harness import filter_eval - - defn = user_tables.get(table_key, st=rt) or {} - fields = [f for f in (defn.get("fields") or []) if isinstance(f, dict)] - by_key = {str(f.get("key")): f for f in fields} - keys = _visible_keys(view, fields) - cfg = (view or {}).get("config") or {} - - # The row pool: the table's own rows, narrowed to the fields it declares, exactly as - # `routes_tables.scoped_pool` builds it for a materialised table. ⚠ A read-through table has - # no rows here and is refused at the MINT, so this branch is the only one that can be reached. - rows_src = [] - for rid, row in (defn.get("rows") or {}).items(): - if not str(rid).isdigit(): - continue - r = {k: v for k, v in (row or {}).items() if k in by_key} - r["pid"] = int(rid) - rows_src.append(r) - rows_src.sort(key=lambda r: r["pid"]) - - # The view's own row selection, through the SHARED evaluator. `_needs_a_reader` has already - # refused anything this context could not answer, so an empty result here means the filter - # genuinely matches nothing — not that we failed to resolve it. - ctx = filter_eval.EvalCtx(today=time.strftime("%Y-%m-%d")) - keep = set(filter_eval.visible_pids(cfg.get("filters"), rows_src, fields, ctx, - member_pids=cfg.get("memberPids"))) - chosen = [r for r in rows_src if r.get("pid") in keep] - - # ⛔⛔ COORDINATES RIDE A MAP WHEN THEY ARE COORDINATES — NOT WHEN THEY ARE VISIBLE, AND NOT - # BECAUSE OF WHAT A COLUMN IS CALLED. Two verifiers, one from each side, are why this reads - # the way it does; the first fix I wrote was wrong and the second report proved it. - # - # ⚠ THE LEAK (verifier #1, driven): a HIDDEN field keyed `lat` holding the string - # `CANARY-LAT-AAA` came out on the wire, because the pair was emitted before the `keys` - # projection and the bypass keyed on the FIELD NAME rather than on the value being a - # coordinate. So a column called `lat` could carry anything — a note, an address — and publish - # it. That is the real defect. - # - # ⛔ MY FIRST FIX GATED ON VISIBILITY, AND IT BROKE THE FEATURE (verifier #2): hiding the raw - # decimals is the NORMAL way somebody builds a Map view — nobody wants `38.7223` in the column - # list — so gating on `visible` meant an ordinary map published a page with no map, under two - # messages that contradicted each other ("no rows carry a location" vs "this view hides its - # location columns"). - # - # ⭐ THE RULE THAT SATISFIES BOTH: publishing a MAP is publishing WHERE THE ROWS ARE — that is - # what the sharer chose — so a real coordinate rides whether or not its column is shown, and a - # value that is not a coordinate never rides at all. `_coord` is the whole wall, and it is a - # VALUE test, so no naming convention can smuggle anything past it. - # ⚠ It mirrors `PublishedView.MapPlot`'s own `coord()` deliberately: the client must not plot - # what the server would not send, and the server must not send what the client would discard. - # Two normalizers on one question is a smell [[one-question-two-normalizers]] — kept here - # because they sit on opposite sides of a trust boundary, where the server's copy is the wall - # and the client's is display hygiene. - mode = _mode_of(view) - plotted = 0 - if mode == "map": - for r in chosen: - if _coord(r.get("lat"), 90) is not None and _coord(r.get("lon"), 180) is not None: - plotted += 1 - - limits = [] - if mode == "map" and chosen and not plotted: - # R6's second sentence. A map with nothing on it must say WHY — and this says the true - # why, which is about the DATA, because visibility is no longer part of the answer. - limits.append({ - "subject": "map", "effect": "not_plotted", - "detail": f"none of these {len(chosen)} rows carry a usable location", - "recommendation": "add `lat` and `lon` values to the records, then reload this link", - }) - if len(chosen) > PUBLIC_ROW_CAP: - # R6's second sentence. A short page that does not say it is short is the silent - # truncation the rule is actually about. - limits.append({ - "subject": "rows", "effect": "windowed", - "detail": f"this view has {len(chosen)} rows and a published page serves the first " - f"{PUBLIC_ROW_CAP}", - "recommendation": "narrow the view's filters, or share it with named people instead " - "of publishing a link", - }) - chosen = chosen[:PUBLIC_ROW_CAP] - - # ⛔ THE DISPLAY REFS ARE INTERSECTED WITH `visible`, NOT UNIONED INTO IT. A Map that colours - # by a column the view HIDES would otherwise put that column's value on every public row — - # the projection leak, arriving through the renderer rather than through the column list. The - # fail-closed choice is to drop the ref and render the map without colour; a published page - # that is slightly plainer beats one that ships a hidden column. - disp_in = (cfg.get("display") or {}) - display = {"mode": mode} - for ref in ("dateField", "stackField", "titleField", "colorField", "sizeField"): - if disp_in.get(ref) in keys: - display[ref] = disp_in[ref] - - return { - "title": str((view or {}).get("name") or "")[:200], - "mode": display["mode"], - "display": display, - "columns": [{"key": k, - "label": str(by_key[k].get("label") or k), - "type": str(by_key[k].get("type") or "text"), - **({"options": [str(o) for o in by_key[k]["options"]][:200]} - if isinstance(by_key[k].get("options"), list) and by_key[k].get("options") - else {})} - for k in keys], - # KEY BY KEY. `pid` rides because the client needs a stable row identity to render a list; - # it is a row NUMBER within this table and names nothing outside it. - # ⛔⛔ `lat`/`lon` RIDE ONLY ON A MAP **AND ONLY WHEN THE VIEW SHOWS THEM** — and the second - # half was missing, which was a LEAK. Found by a verifier that drove this route with a - # hidden field keyed `lat` carrying the string `CANARY-LAT-AAA`, and watched it come out - # on the wire. - # - # The first version emitted the pair BEFORE the `keys` projection, so `_visible_keys` never - # gated it. On a `ut_*` table coordinates are not magic: `routes_tables.scoped_pool` builds - # its row as `{k: v for k, v in row.items() if k in field_keys}`, so a value only survives - # if the table DECLARES a field keyed `lat`/`lon` — i.e. they are ORDINARY COLUMNS, and a - # view can hide them like any other. Hiding them therefore has to work here, because on a - # published page **the projection is the only wall there is** (`ut_*` databases have no - # hidden-field closure behind it, `routes_shares.py`'s docstring). - # - # ⚠ The bypass was keyed on the FIELD NAME, never on the value being a coordinate, so it - # forwarded whatever a column called `lat` happened to hold — a string, a note, anything. - # ⚠ And the shipped gate could not see it: it asserted the key NAMES rode on a map and not - # on a catalog, over a fixture whose rows carried no `lat` key at all — so it pinned the - # names while both values were `None` [[gate-answers-the-wrong-question]]. - # KEY BY KEY. `pid` rides because the client needs a stable row identity; it is a row - # NUMBER within this table and names nothing outside it. `lat`/`lon` ride only on a map, - # and only when they PARSE as coordinates — see the block above for why that is the test. - "rows": [{"pid": r.get("pid"), - **({"lat": _coord(r.get("lat"), 90), "lon": _coord(r.get("lon"), 180)} - if mode == "map" - and _coord(r.get("lat"), 90) is not None - and _coord(r.get("lon"), 180) is not None else {}), - **{k: r.get(k) for k in keys}} for r in chosen], - "total": len(chosen), - **({"limits": limits} if limits else {}), - } - - -async def _bounded_body(request: Request) -> dict: - """The request body, read WITH A BOUND — never `Body(...)`, never `await request.body()`. - - FastAPI reads and JSON-parses the WHOLE body before the handler's first line runs, so a - declared model is not a bound at all; `content-length` is caller-supplied, so checking it is - not one either. Streaming with a running total is the actual bound. - """ - size, chunks = 0, [] - async for chunk in request.stream(): - size += len(chunk) - if size > MAX_BODY_BYTES: - raise err(413, "body_too_large", "that request is too large") - chunks.append(chunk) - import json - try: - parsed = json.loads(b"".join(chunks) or b"{}") - except ValueError: - raise err(400, "bad_request", "that request could not be read") - return parsed if isinstance(parsed, dict) else {} - - -# ── THE SHARER'S DOOR (authenticated, creator-or-admin) ────────────────────────────────────── -# -# ⚠ THE NOUN IS `/view-link`, DELIBERATELY NOT `/views/{...}/publish`, and it mirrors the form -# door's `/form-link` for the same reason: a literal path segment sitting beside a `/{token}` -# wildcard is resolved by DECLARATION ORDER, and a noun that cannot collide with a token has no -# order to get wrong. - - -@router.get("/view-link") -def read_view_link(topic: str = "", view: str = "", - session: Session = Depends(require_session)): - """This view's link state. ⛔ A GET MUST NOT MINT — opening the panel is not publishing.""" - v, _mode = _may_administer_view(session, str(topic or ""), str(view or "")) - token, entry = _entry_for(session.runtime, str(topic), str(view)) - out = _state_of(token, entry) - # The presentational flags contract C1 put on the view spec, echoed back so the client can - # tell whether the two agree. They are a MIRROR of this bucket, never its source. - disp = ((v.get("config") or {}).get("display") or {}) if isinstance(v, dict) else {} - out["displayPublished"] = disp.get("published") is True - return out - - -@router.post("/view-link") -async def mint_view_link(request: Request, session: Session = Depends(require_session)): - """Publish this view, or change its access. `{topic, view, access?, passphrase?, rotate?}`. - - IDEMPOTENT WITHOUT `rotate`: opening the panel twice, or switching public↔password, must not - invalidate a link somebody already sent. `rotate: true` mints a fresh token and the previous - one dies in the SAME write, so there is never a window where both open the view. - """ - body = await _bounded_body(request) - topic, view = str(body.get("topic") or ""), str(body.get("view") or "") - _v, _mode = _may_administer_view(session, topic, view) - - access = "public" if body.get("access") == "public" else "password" - raw_pw = body.get("passphrase") - passphrase = "" if raw_pw is None else str(raw_pw) - if len(passphrase) > MAX_PASSPHRASE: - raise err(400, "passphrase_too_long", - f"a passphrase can be at most {MAX_PASSPHRASE} characters") - - existing_token, existing = _entry_for(session.runtime, topic, view) - rotate = bool(body.get("rotate")) - fresh = secrets.token_urlsafe(TOKEN_BYTES) - - # ⛔ A `password` LINK MUST END UP WITH A HASH, and there are exactly two ways to have one: - # the caller supplied a passphrase now, or one was already stored and is being kept. Anything - # else is refused HERE rather than stored and refused later — a link that cannot be opened by - # anybody is not a safe default, it is a broken feature that reads as a permission bug. - if access == "password": - if passphrase and len(passphrase) < MIN_PASSPHRASE: - raise err(400, "passphrase_too_short", - f"a passphrase needs at least {MIN_PASSPHRASE} characters") - if not passphrase and not (existing or {}).get("pw"): - raise err(400, "passphrase_required", - "a password-protected link needs a passphrase") - - minted = {"token": existing_token or fresh} - - def _set(cur): - cur = dict(cur or {}) - prev = None - for stored, where in list(cur.items()): - if (isinstance(where, dict) and where.get("table") == topic - and where.get("view") == view): - prev = dict(where) - cur.pop(stored, None) - if not rotate: - minted["token"] = str(stored) - if rotate or prev is None: - minted["token"] = fresh - entry = {"table": topic, "view": view, "access": access, - "createdBy": str((prev or {}).get("createdBy") or session.uname), - "createdAt": float((prev or {}).get("createdAt") or time.time())} - # ⛔ THE STORED PASSPHRASE SURVIVES A TRIP THROUGH `public`, and the first version DROPPED - # it — found by a verifier that traced the toggle rather than the happy path. Publishing - # as `public` skipped this block entirely, so the hash was destroyed while the TOKEN was - # kept; switching back to `password` then demanded a new passphrase, silently, while the - # panel's own sentence promised the opposite ("leave blank to keep the current one"). - # ⚠ Carrying it is inert, not lax: `_pw_ok` is consulted ONLY when `access == "password"` - # (`get_published`/`open_published` both test it first), and `_state_of` exposes a boolean, - # never the digest. A hash nobody can reach is not a secret in use — but a promise the UI - # makes and the store breaks is a defect either way, and the honest fix is to keep the - # promise rather than to reword it. - if passphrase: - salt = secrets.token_bytes(PW_SALT_BYTES) - entry["salt"] = salt.hex() - entry["iter"] = PW_ITERATIONS - entry["pw"] = _hash_pw(passphrase, salt) - elif (prev or {}).get("pw"): - # Carried key by key, so a future field on the entry is not silently inherited. - entry["salt"] = str((prev or {}).get("salt") or "") - entry["iter"] = int((prev or {}).get("iter") or PW_ITERATIONS) - entry["pw"] = str((prev or {}).get("pw") or "") - cur[minted["token"]] = entry - return cur - - session.runtime.update(TOKENS_KEY, _set, flush="sync") - _mirror_display(session.runtime, topic, view, True, access) - token, entry = _entry_for(session.runtime, topic, view) - # ⛔ NO FABRICATED FALLBACK ENTRY HERE. The first version answered - # `_state_of(token or minted["token"], entry or {"access": access, "pw": "x"})`, and that - # `"pw": "x"` would have reported `hasPassphrase: true` about a PUBLIC link if the read-back - # ever came back empty — a lie in the safe-looking direction, which is the kind that survives. - # If the write cannot be read back, say so; do not describe a state nobody verified. - if not entry: - raise err(503, "not_saved", - "the link was minted but could not be read back — reload and try again") - return _state_of(token, entry) - - -@router.delete("/view-link") -async def revoke_view_link(request: Request, session: Session = Depends(require_session)): - """Unpublish. `{topic, view}`. - - ⛔ REVOKE ROTATES — it does not merely unset a flag. The token STRING is dropped from the - index in this write, so the old link stops resolving immediately; and because a later publish - mints `secrets.token_urlsafe(24)` afresh, the revoked string can never come back. An - implementation that kept the token and flipped an `enabled` flag would leave the secret live - in the store, one bug away from working again, and would make "revoked" a property somebody - could forget to check on a code path added later. - """ - body = await _bounded_body(request) - topic, view = str(body.get("topic") or ""), str(body.get("view") or "") - _may_administer_view(session, topic, view) - - def _drop(cur): - cur = dict(cur or {}) - for stored, where in list(cur.items()): - if (isinstance(where, dict) and where.get("table") == topic - and where.get("view") == view): - cur.pop(stored, None) - return cur - - session.runtime.update(TOKENS_KEY, _drop, flush="sync") - # ⛔ AND THE MIRROR COMES DOWN IN THE SAME BREATH. Without this the grid goes on showing - # "published" about a link that no longer resolves — the half a reviewer caught. - _mirror_display(session.runtime, topic, view, False) - return {"published": False, "access": "password", "token": "", "url": ""} - - -# ── THE PUBLIC DOOR (no session — this is the whole feature) ───────────────────────────────── -# -# ⛔ NO `Depends(require_session)` ON EITHER ROUTE BELOW, DELIBERATELY. "Public" in this app is not -# a flag or an allow-list entry — `main.py` has no auth middleware and no exempt-path table; a -# route is public exactly by omitting the dependency. Which is why the two are kept together, -# under one banner, rather than filed beside the sharer's routes they resemble. - - -@router.get("/published/{token}") -def get_published(token: str, request: Request): - """The published view. Read-only, unauthenticated, projected to the view's own columns.""" - if not _rate_ok(_client_ip(request), time.time()): - raise err(429, "too_many_requests", "too many requests — wait a moment and try again") - found = _resolve(token) - if not found: - # ⛔⛔ W33-T70 — AN UNKNOWN TOKEN ANSWERS EXACTLY WHAT A LOCKED ONE ANSWERS. - # This used to `raise _refuse()`, and that 403 was a free oracle: one unauthenticated GET, - # no passphrase, no cost, told an enumerator whether a token was REAL. `_refuse`'s own - # docstring says the difference between "no such link" and "that link was revoked" must - # never be observable — and the route beside it published that difference in its status - # code. Sorting real tokens from fake ones is the whole of the work; once it is free, the - # passphrase is all that is left and it can be attacked offline-cheap. - # ⚠ SO THE UNKNOWN TOKEN GETS THE LOCKED SHAPE: `{"locked": true}` and nothing else — no - # title, no columns, no count, the same bytes a real password link returns before anyone - # has tried to open it. The guess then costs a POST with a passphrase, which is rate - # limited and PBKDF2-priced. A PUBLIC link still opens on this GET, which is what a public - # link is for; what stops being visible is which PASSWORD tokens exist. - _note_failure(_client_ip(request), time.time()) - _equalise_pw_cost(None) - return {"locked": True} - rt, _slug, table_key, view, entry = found - if entry.get("access") == "password": - # ⛔ THE SHAPE OF THE PASSWORD ANSWER, and it is not a refusal. A locked link must render - # a passphrase prompt, so this 200 says "there is something here and it is locked" and - # NOTHING else — no title, no column names, no row count. A 403 here would be - # indistinguishable from a bad token, which is right for a WRONG passphrase and wrong for - # a link the holder has not tried to open yet. - return {"locked": True} - return _public_view(rt, table_key, view) - - -@router.post("/published/{token}") -async def open_published(token: str, request: Request): - """Open a password-protected link. `{passphrase}`. - - ⛔ A WRONG PASSPHRASE AND A WRONG TOKEN ANSWER THE SAME 403, from the same `_refuse`. If they - differed, the route would confirm which tokens are real to anybody willing to send one guess — - and a 24-byte token's entire protection is that it cannot be found by guessing. - ⚠ The passphrase is compared against a PBKDF2 digest with `hmac.compare_digest`, and is never - stored, logged or echoed. D-130's scar: a form's invited-address list is IDENTIFICATION and - anyone may claim an identity; this is AUTHENTICATION and is treated as one. - """ - now = time.time() - if not _rate_ok(_client_ip(request), now): - raise err(429, "too_many_requests", "too many requests — wait a moment and try again") - body = await _bounded_body(request) - found = _resolve(token) - if not found: - # ⛔⛔ W33-T70 — SPEND THE PBKDF2 ANYWAY. A wrong token skipped the hash entirely and - # answered in ~0.4 ms while a wrong passphrase paid ~82 ms: a 206x gap, zero overlap in 24 - # samples, and a clean separation of real tokens from fake ones for anyone with a stopwatch. - # Both arms now cost the same, so the identical 403 above is finally identical in practice - # rather than only in wording. - _note_failure(_client_ip(request), now) - _equalise_pw_cost(None) - raise _refuse() - rt, _slug, table_key, view, entry = found - if entry.get("access") == "password" and not _pw_ok(entry, str(body.get("passphrase") or "")): - _note_failure(_client_ip(request), now) - raise _refuse() - # ⚠ A link with NO passphrase must still pay, or "this token is public" is readable from the - # clock on a route whose whole job is to be uninformative. - if entry.get("access") != "password": - _equalise_pw_cost(entry) - return _public_view(rt, table_key, view) +"""routes_publish.py — PUBLISH AN INTERFACE VIEW AS A LINK (wave 33, owner item 8b, ruling R5). + +The owner, verbatim: *"Have the ability to publish interface, when a user's view is an interface +(e.g. Map or Catalog), we should have the ability to publish a link that can be either accessed +publicly or with a password that the user that share it can toggle."* + +R5, in five clauses, and every one of them is load-bearing: + * publish = a per-view secret token in a **server-only bucket**; + * a `public | password` toggle the SHARER owns, with a **hashed** passphrase beside it; + * an **unauthenticated** read-only route plus a `#/v/` client route; + * the publish surface **projects only the columns the view shows** — never the whole table; + * **creator-or-admin** may publish, and **revoke ROTATES** the token. + +⛔ THIS FILE COPIES `routes_forms.py`'S CONSTRUCTION ON PURPOSE, AND THE TICKET SAID TO. That door +has been public since wave 23 and carries scar tissue no fresh design would reproduce: ONE refusal +for every resolution failure (so the route is not an oracle for which tokens are real), a +constant-time compare, a STREAMING body bound rather than `Body(...)` (FastAPI reads and parses the +whole body before the handler's first line, and `content-length` is caller-supplied), a sliding +rate window rather than a fixed bucket, and an index that is a POINTER, never a permission. + +⚠ WHAT IS DELIBERATELY *NOT* SHARED WITH `routes_forms.py`: the bucket. A form token buys a WRITE +door into a table; a publish token buys a READ projection of one view. Folding them into one index +would make a single leaked string ambiguous about which of the two it opens, and would make +`_resolve` return an object whose capability depends on a field rather than on which door was +knocked. Two buckets, two resolvers, one shape. + +⚠ AND NOT SHARED WITH SHARING. **Three systems now answer some version of "who can see this"** and +they are not the same question (audit S-4): the grant registry drives *Shared with me*, +`table_store.is_shared` GATES opening a view inside the app, and a published link is a THIRD thing +— an unauthenticated read of a projection, bound to a secret rather than to an account. A published +link is NOT a grant: it creates no registry row, appears in nobody's *Shared with me*, and cannot +be revoked by removing a person, because there is no person. + + python verify_forms.py # this file's gate; section 6 onward +""" +import hashlib +import hmac +import os +import secrets +import time + +from fastapi import APIRouter, Depends, Request + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + +#: The SERVER-ONLY index: `{token: {table, view, access, pw?, salt?, iter?, createdBy, createdAt}}`. +#: ⛔ IT IS NEVER `display.*`. Contract C1 puts two PRESENTATIONAL flags on the view spec +#: (`published`, `publishAccess`) precisely so the client has something to render, and the browser +#: writes that spec on every autosave — so anything secret living there would be echoed back to the +#: browser by construction. `aios_grid._clean_display`'s allowlist enforces the other half. +TOKENS_KEY = "publish_tokens" +TOKEN_BYTES = 24 + +#: Passphrase storage. PBKDF2-HMAC-SHA256 with a per-link salt. +#: ⚠ D-130 IS THE SCAR THIS AVOIDS: a form's "invited addresses" list is IDENTIFICATION — it says +#: who you claim to be and anyone may claim it. A passphrase is AUTHENTICATION. The difference is +#: not a stronger string, it is that the secret is never stored, never logged and never echoed. +PW_ITERATIONS = 240_000 +PW_SALT_BYTES = 16 +MIN_PASSPHRASE = 6 +MAX_PASSPHRASE = 128 + +#: v1 request protections, the same shape and the same numbers as the form door (contract C9), so +#: the two public routes cannot drift into different postures. ⚠ In-process, therefore PER WORKER. +RATE_WINDOW_S = 60 +RATE_PER_WINDOW = 30 +MAX_BODY_BYTES = 16 * 1024 + +_HITS: dict = {} + +#: The view kinds that may be published, i.e. R5's "interface". +#: ⚠ THE CLIENT'S SOURCE OF TRUTH IS `customer-grid/iconShapes.ts::MODE_GROUP` (wave 33 item 9 +#: moved `swipe` and `timeseries` into this group). This constant is the SERVER's copy and the two +#: are held in step by `verify_forms.py`, which parses `MODE_GROUP` and compares — because a +#: server list that silently drifts from the picker means a mode a user can create and cannot +#: publish, with nothing anywhere going red. A `grid` or `kanban` view is a re-shaping of a row +#: set; publishing one would be publishing the table, which is exactly what R5's projection clause +#: exists to prevent. +#: ⛔⛔ W33-T68 — `form` IS DELIBERATELY ABSENT, AND ITS ABSENCE IS THE FIX. +#: A `form` view's rows ARE the submissions people have sent it. Every other mode here re-shapes a +#: row set the publisher already curated; a form's row set is other people's answers, gathered under +#: an implicit promise that they go to the owner. Publishing one turned "share this interface" into +#: "serve the responses to anyone holding the link" — on the one unauthenticated door in the +#: product, with no wall between the link and the data. +#: ⚠ AND A FORM ALREADY HAS ITS OWN PUBLIC DOOR: `#/form/` through `routes_forms.py`, which +#: serves the BLANK form for submitting and never the stored rows. So this is not a capability +#: removed, it is a second door onto the same object that should never have existed beside the +#: first. Publishing a form to be filled in still works, at the URL that was always for it. +#: ⚠ `verify_forms.py` holds this list in step with the client's `MODE_GROUP`; the client must not +#: offer Publish on a form view, or the picker promises what this refuses. +PUBLISHABLE_MODES = ("map", "catalog", "swipe", "timeseries") + + +def _refuse(): + """THE ONE REFUSAL, for every way a published link can fail to resolve. + + Wrong token, revoked link, deleted view, deleted table, a view that stopped being an + interface, a wrong passphrase. ⛔ Callers must not branch a more specific message out of it: + the difference between "no such link" and "that link was revoked" tells an enumerator which + tokens are real, and the difference between "no such link" and "wrong passphrase" tells them + which links are worth guessing at. + """ + return err(403, "bad_publish_token", "that link is not valid") + + +def _same(a: str, b: str) -> bool: + """Constant-time. A `==` leaks a secret's prefix through timing, one character at a time.""" + return hmac.compare_digest(str(a or ""), str(b or "")) + + +def _client_ip(request: Request) -> str: + """⛔⛔ W33-T70 — `x-forwarded-for` IS GONE FROM THIS FUNCTION, AND THAT WAS THE WHOLE HOLE. + + The header is chosen by the caller. Keying a rate limit on it means an enumerator writes a new + value per request and every request lands in a fresh bucket: MEASURED at **100 of 100 admitted + through a 30-per-window ceiling**. A limiter with a caller-chosen key is not a limiter, and its + old docstring said the header was "a rate-limit key and NOTHING else" — which was exactly the + use it could not support. It is safe as a LOG field and as nothing else. + + The socket peer is the only thing here the caller cannot choose, so it is the key. + ⚠ AND BEHIND HF'S PROXY EVERY CALLER SHARES ONE PEER, which is why `_rate_ok` counts FAILURES + ONLY (see there). A per-peer ceiling over ALL traffic would be one global bucket, i.e. an alarm + that fires for everyone [[alarm-that-fires-for-everyone]] — the honest reading of a shared peer + is that we cannot separate callers, not that we should throttle them together. + """ + return request.client.host if request.client else "?" + + +def _rate_ok(key: str, now: float) -> bool: + """Is this caller UNDER the failure ceiling? Pure check — call `_note_failure` to count. + + A SLIDING window. A fixed bucket lets a caller spend a whole allowance at 11:59:59 and the + whole next one at 12:00:00 — i.e. double the limit, back to back, against a door whose entire + protection is that guessing a 24-byte token is slow. + + ⛔ W33-T70 — IT COUNTS FAILURES, NOT REQUESTS, and the split is what makes a shared proxy peer + survivable. A legitimate reader opens a link that RESOLVES, so they never touch the counter at + all; an enumerator produces nothing but misses. Counting every request under one shared peer + would have denied service to everybody the moment one guesser showed up — trading a token + oracle for an outage is not a fix. + """ + seen = [t for t in _HITS.get(key, ()) if now - t < RATE_WINDOW_S] + _HITS[key] = seen + return len(seen) < RATE_PER_WINDOW + + +def _note_failure(key: str, now: float) -> None: + """Record one failed resolution against this peer, and keep the table bounded.""" + seen = [t for t in _HITS.get(key, ()) if now - t < RATE_WINDOW_S] + seen.append(now) + _HITS[key] = seen + if len(_HITS) > 4096: + for k in [k for k, v in _HITS.items() if not v or now - v[-1] > RATE_WINDOW_S]: + _HITS.pop(k, None) + + +#: A salt used ONLY to burn the same PBKDF2 a real verification would, when there is no stored hash +#: to check against. Its value is irrelevant; its COST is the point. +_DUMMY_SALT = b"\x00" * 16 + + +def _equalise_pw_cost(entry) -> None: + """⛔⛔ W33-T70 — SPEND THE PBKDF2 EVEN WHEN THERE IS NOTHING TO VERIFY. + + MEASURED before this existed: a wrong TOKEN answered in ~0.4 ms and a wrong PASSPHRASE in + ~82 ms — a **206x gap with zero overlap across 24 samples**. So the refusal's careful wording + ("that link is not valid", identical for both) was undone by the clock: anyone could sort real + tokens from fake ones by timing alone, then spend their guesses only on the real ones. The + docstring on `_refuse` describes exactly the leak the code then handed over for free. + + Called on every path that fails BEFORE a passphrase check would have happened, so the cheap + branch costs what the expensive one costs. `iter` is taken from the entry when there is one, so + a link minted under a different iteration count stays indistinguishable too. + """ + iterations = PW_ITERATIONS + if isinstance(entry, dict): + try: + iterations = int(entry.get("iter") or PW_ITERATIONS) + except (TypeError, ValueError): + iterations = PW_ITERATIONS + _hash_pw("", _DUMMY_SALT, iterations) + + +def _public_base() -> str: + return (os.environ.get("AIOS_PUBLIC_BASE") or os.environ.get("APP_BASE_URL") or "").rstrip("/") + + +def _link_of(token: str) -> str: + return f"{_public_base()}/#/v/{token}" + + +def _tokens(rt) -> dict: + """This tenant's publish index, always a dict.""" + try: + found = rt.get(TOKENS_KEY) or {} + except Exception: # noqa: BLE001 + return {} + return found if isinstance(found, dict) else {} + + +def _hash_pw(passphrase: str, salt: bytes, iterations: int = PW_ITERATIONS) -> str: + return hashlib.pbkdf2_hmac("sha256", str(passphrase).encode("utf-8"), + salt, int(iterations)).hex() + + +def _pw_ok(entry: dict, passphrase: str) -> bool: + """Constant-time verify against the stored digest. + + ⛔ RETURNS FALSE, NEVER RAISES, and never distinguishes "this link has no passphrase stored" + from "the passphrase is wrong" — a `password` link whose hash went missing must refuse, not + fall open. [[default-must-pass-its-own-guard]] + """ + stored, salt_hex = str(entry.get("pw") or ""), str(entry.get("salt") or "") + if not stored or not salt_hex: + return False + try: + salt = bytes.fromhex(salt_hex) + except ValueError: + return False + got = _hash_pw(passphrase, salt, int(entry.get("iter") or PW_ITERATIONS)) + return _same(got, stored) + + +def _find_view(rt, table_key: str, view_id: str): + """`(view, owner_username)` from the table's workspace bucket, or `(None, "")`. + + The bucket shape is `{username: {views: {id: view}}}` — the same walk `routes_forms._find_view` + does, and the reason a view is addressable by id ALONE here: the id is unique within the table, + while the owner is what we are trying to discover. + """ + if not table_key or not view_id: + return None, "" + try: + bucket = rt.get(f"{table_key}_table_workspace") or {} + except Exception: # noqa: BLE001 + return None, "" + if not isinstance(bucket, dict): + return None, "" + for owner, blob in bucket.items(): + views = (blob or {}).get("views") if isinstance(blob, dict) else None + if isinstance(views, dict) and isinstance(views.get(view_id), dict): + return views[view_id], str(owner) + return None, "" + + +def _mode_of(view: dict) -> str: + """The view's display mode. Absent means `grid` — the stored default is "say nothing".""" + cfg = (view or {}).get("config") if isinstance(view, dict) else None + disp = (cfg or {}).get("display") if isinstance(cfg, dict) else None + return str((disp or {}).get("mode") or "grid") if isinstance(disp, dict) else "grid" + + +def _may_administer_view(session: Session, table_key: str, view_id: str): + """`(view, mode)` when this caller may publish THIS view, else raises. R5's creator-or-admin. + + ⛔ `session.uname` / `session.admin`, NOT `username` / `is_admin`. `Session` has no such + attributes, and reading one raises ABOVE this route's own `try:` — which is how D-107 arrived + as a bare plain-text 500 rather than as our JSON envelope. + """ + import core.table_store as table_store + + if not table_key.startswith("ut_"): + # The Odoo/registry grids are read-through mirrors with their own permission wall; a + # published projection of one would be a second, secret-gated door onto tenant data whose + # visibility the module gate is supposed to decide. + raise err(400, "not_a_database", "you can publish views on your own databases only") + # ⛔ A READ-THROUGH DATABASE CANNOT BE PUBLISHED, AND THE REFUSAL SAYS SO RATHER THAN MINTING + # A LINK THAT WILL NOT OPEN. `startswith("ut_")` admits `ut_odoo_*` and `ut_meta_*`, whose rows + # do not live in the tenant document at all — they are windowed out of the DuckDB mirror + # through a Session-bound path (`ut_assembly`, which 409s `window_required` on the big ones, + # D-174's lineage). An unauthenticated route has no session and therefore no such path, so a + # token minted here would resolve to a page that could never render. + # ⚠ THIS IS R6's SECOND SENTENCE, WHICH IS THE HALF THAT GETS DROPPED: a limit that genuinely + # cannot be removed must be REPORTED, with its cause, never silently enforced. Refusing at the + # MINT — where a person is standing in front of the answer — is the only place that reads as a + # sentence rather than as an empty page. + import core.user_tables as user_tables + + if user_tables.is_connected(table_key, st=session.runtime): + raise err(400, "connected_source", + "a database that reads through a connected source (Odoo, Meta Ads) cannot be " + "published as a public link: its rows are served from the tenant's mirror by a " + "signed-in request, and a public link has no session to serve them with") + view, owner = _find_view(session.runtime, table_key, view_id) + if not isinstance(view, dict): + raise err(404, "no_such_view", "that view no longer exists") + if not (owner == session.uname + or table_store._may_administer(view, session.uname, session.admin)): + raise err(403, "not_yours", "only this view's creator or an admin can publish it") + mode = _mode_of(view) + if mode not in PUBLISHABLE_MODES: + raise err(400, "not_an_interface", + "only an interface view (Map, Catalog, Swipe, Time-series, Form) can be " + "published as a link") + cfg = view.get("config") or {} + if cfg.get("cohortLock"): + raise err(400, "reader_scoped", + "this view is locked to a cohort, and a cohort's membership is resolved for " + "the person reading it — a public link has no reader, so the page would show " + "no rows at all. Publish a copy without the cohort lock.") + if _needs_a_reader(cfg.get("filters")): + raise err(400, "reader_scoped", + "this view filters on a cohort, a measure rule or a top-N slice, and each of " + "those is resolved for the person reading it — a public link has no reader, so " + "the page would show no rows at all. Publish a copy filtered on columns.") + return view, mode + + +#: How many rows a published page will serve. ⚠ R6 (no cap on connected-source data) does not +#: reach here twice over: a publishable database is `records_mutable`, i.e. the EDITABLE substrate +#: that keeps its bound (`user_tables.MAX_ROWS`), and a connected one is refused at the mint. What +#: R6's SECOND sentence does reach here is the reporting duty — a page that serves fewer rows than +#: the view has must SAY SO on the wire, with the cause, never just stop. +PUBLIC_ROW_CAP = 5000 + + +def _needs_a_reader(tree) -> bool: + """Does this filter tree contain a leaf that only a SIGNED-IN reader could resolve? + + ⛔ COHORTS, MEASURE RULES AND RANK SLICES ARE NOT PROPERTIES OF THE VIEW. Each is a SET the + host computes for the person asking — `filter_eval.EvalCtx`'s own docstring: *"each is an + answer a single ROW cannot compute … absent, the condition matches NOTHING rather than + everything"*. That default is right (it fails closed) and it is unusable here: an anonymous + page whose every row was silently filtered out is indistinguishable from a broken link, and + the reader has nobody to ask. Worse, `rank_sets` has NO server-side resolver anywhere in this + repo — `routes_alerts._evaluate` passes cohort/measure/today and nothing else — so a `topN` + view would serve zero rows on the server while showing twenty in the browser. + + ⭐ So such a view is refused AT THE MINT, where a person is standing in front of the answer. + That is R6's second sentence: a limit that genuinely cannot be removed is REPORTED with its + cause, never silently enforced. + + ⚠ THE PREDICATES ARE `filter_eval`'S OWN. Re-implementing "is this a cohort leaf?" here would + be a second evaluator that agrees today and drifts the day the leaf shape changes — and it + would drift SILENTLY, because the two answers only differ on views nobody has published yet. + [[one-evaluator-per-question]] + """ + from harness import filter_eval + + def walk(node) -> bool: + if isinstance(node, list): + return any(walk(n) for n in node) + if not isinstance(node, dict): + return False + if node.get("kind") == "group" or isinstance(node.get("children"), list): + return any(walk(n) for n in (node.get("children") or [])) + if filter_eval._is_cohort(node) or filter_eval._is_measure(node): + return True + return node.get("op") in filter_eval.RANK_OPS + + return walk(tree) + + +def _mirror_display(rt, table_key: str, view_id: str, published: bool, access: str = "password"): + """Keep contract C1's two PRESENTATIONAL flags on the stored view in step with this bucket. + + ⛔ WHY THIS EXISTS AT ALL, and it was found by a reviewer rather than by a gate: revoking a + link dropped the token and left `config.display.published: true` on the view, so the grid's + own UI would go on saying "published" about a link that no longer resolves. Two records of one + fact, and only one of them moving, is the [[flag-shipped-without-its-writer]] shape — here with + the writer present and the OTHER half forgotten. + + ⚠ THE BUCKET REMAINS THE TRUTH. These flags exist so the client has something to render + without asking; they are a MIRROR, never a source, and nothing in this module reads them back + to decide anything. `read_view_link` deliberately reports both so a drift is visible rather + than assumed away. + + ⚠ ONLY THE TWO LEGAL KEYS ARE WRITTEN, with E's fail-closed coercion reproduced exactly + (`aios_grid._clean_display`: `published: True` always carries a `publishAccess`, and anything + that is not the literal `public` stores as `password`) — so a value written here and a value + written by the browser cannot disagree. + """ + def _set(cur): + cur = dict(cur or {}) + for owner, blob in list(cur.items()): + views = (blob or {}).get("views") if isinstance(blob, dict) else None + view = views.get(view_id) if isinstance(views, dict) else None + if not isinstance(view, dict): + continue + cfg = dict(view.get("config") or {}) + disp = dict(cfg.get("display") or {}) + if not disp.get("mode"): + # No display block means no interface view; nothing here should invent one. + continue + if published: + disp["published"] = True + disp["publishAccess"] = "public" if access == "public" else "password" + else: + disp.pop("published", None) + disp.pop("publishAccess", None) + cfg["display"] = disp + cur[owner] = {**blob, "views": {**views, view_id: {**view, "config": cfg}}} + return cur + + try: + rt.update(f"{table_key}_table_workspace", _set, flush="sync") + except Exception: # noqa: BLE001 + # ⚠ SWALLOWED, and deliberately: the token bucket is the truth and it has already been + # written. A mirror that failed to update leaves the UI one refresh out of date, which is + # strictly better than a 500 on a publish that actually succeeded. + pass + + +def _entry_for(rt, table_key: str, view_id: str): + """`(token, entry)` for a view's existing link, or `(None, None)`.""" + for token, where in _tokens(rt).items(): + if (isinstance(where, dict) and where.get("table") == table_key + and where.get("view") == view_id): + return str(token), where + return None, None + + +def _state_of(token, entry) -> dict: + """The link's state as the SHARER may see it. Built key by key. + + ⛔ NO `**entry`. The stored blob carries `pw` and `salt`; a spread would put both on the wire + to the browser, which is the whole failure this file's bucket exists to avoid, and it would do + it silently the first time somebody added a field. + """ + if not token or not isinstance(entry, dict): + return {"published": False, "access": "password", "token": "", "url": ""} + return { + "published": True, + "access": "public" if entry.get("access") == "public" else "password", + "token": str(token), + "url": _link_of(str(token)), + # A boolean, never the digest and never the salt. + "hasPassphrase": bool(entry.get("pw")), + "createdBy": str(entry.get("createdBy") or ""), + } + + +def _resolve(token: str): + """`(runtime, tenant_slug, table_key, view, entry)` for a token, or None. + + ⛔ THE INDEX IS A POINTER, NEVER A PERMISSION. Holding a token means the sharer minted it for + THIS view; it does not mean the holder may read anything else, and nothing downstream of here + may widen the subject beyond the `(table, view)` pair the entry names. + + ⚠ IT WALKS EVERY TENANT, because an unauthenticated request carries no tenant. That is the + form door's shape too, and the reason `_same` is constant-time: the walk compares the caller's + string against every stored token in the deployment. + """ + from harness import runtime as _rt + + if not token or len(token) < 16: + return None + for slug in _rt.known_tenants(): + try: + rt = _rt.get_runtime(slug) + except Exception: # noqa: BLE001 + continue + for stored, where in _tokens(rt).items(): + if not _same(str(stored), token) or not isinstance(where, dict): + continue + table_key = str(where.get("table") or "") + view, _owner = _find_view(rt, table_key, str(where.get("view") or "")) + # A view deleted, or re-saved as a grid, since the link was minted. Both answer the + # ONE refusal — "that link is not valid" — rather than explaining which. + if not table_key or not isinstance(view, dict): + return None + if _mode_of(view) not in PUBLISHABLE_MODES: + return None + return rt, slug, table_key, view, where + return None + + +def _coord(value, limit: float): + """A real coordinate, or `None`. THE WALL that lets a map publish without publishing a column. + + ⛔ A VALUE TEST, NEVER A NAME TEST. A field merely CALLED `lat` proves nothing — a verifier put + the string `CANARY-LAT-AAA` in one and watched it reach the wire under the first version of + this code. Anything that is not a finite number inside the earth's range is not a location and + does not travel. + ⚠ `bool` is rejected explicitly: `isinstance(True, int)` is True in Python, and `float(True)` + is `1.0` — a checkbox column named `lat` would otherwise publish as a point off the coast of + Ghana. + """ + if value is None or isinstance(value, bool): + return None + try: + n = float(value) + except (TypeError, ValueError): + return None + return n if n == n and abs(n) <= limit else None + + +def _visible_keys(view: dict, fields: list) -> list: + """The columns this view SHOWS, in the view's own order. R5's projection clause, in one place. + + ⛔ `config.order` IS NOT THE ANSWER AND IS THE OBVIOUS WRONG ONE. `aios_grid._default_view_config` + builds `order` as `shown + hidden`, and `grid_events`' `view_upsert` APPENDS every remaining + field to it — so `order` is every column the table has, hidden ones included. `visible` is the + only allowlist that exists; there is no stored "hidden" key to subtract. + + ⚠ AND AN EMPTY `visible` IS NOT "NO COLUMNS". `CustomerGrid` falls back to the table's default + set when a saved view carries none, so a public page that read `[]` as an empty allowlist would + render blank — and one that read it as "all fields" would LEAK. The fallback is the same + predicate the default config uses (`field.default is not False`), taken from `aios_grid` rather + than restated here. + """ + import aios_grid + + by_key = {str(f.get("key")): f for f in fields if isinstance(f, dict)} + stored = [str(k) for k in (((view or {}).get("config") or {}).get("visible") or [])] + keep = [k for k in stored if k in by_key] + if keep: + return keep + # ⛔⛔ W33-T69 — A STALE `visible` MUST SERVE NOTHING, NOT THE TABLE DEFAULT. + # `delete_field` never prunes a view's stored `visible`, so a published view whose columns were + # later deleted and replaced arrives here with a NON-EMPTY `stored` of which nothing survives — + # and the fallback below then WIDENS the public payload to whatever the table declares by + # default. The publisher chose five columns; the anonymous reader gets the table's idea of + # sensible. That is a widening on the one unauthenticated door in the product. + # ⚠ THE DISTINCTION IS `stored` NON-EMPTY, NOT `keep` EMPTY. A view that never stored `visible` + # at all (a legacy publish, a view saved before the key existed) has no intent to honour and + # the default IS the right answer for it — that is what the fallback was written for. A view + # that stored five keys and has none left DID state an intent, and every column it named is + # gone: the honest answer is no columns, which renders as an empty published view rather than + # somebody else's data. + if stored: + return [] + try: + default_visible = (aios_grid._default_view_config(fields) or {}).get("visible") or [] + except Exception: # noqa: BLE001 + default_visible = [] + fallback = [str(k) for k in default_visible if str(k) in by_key] + return fallback or [str(f.get("key")) for f in fields if f.get("default") is not False] + + +def _public_view(rt, table_key: str, view: dict) -> dict: + """The ONLY bytes a valid token buys. Built KEY BY KEY — there is no `**row` in this function. + + ⛔ THE REASON THAT IS A RULE AND NOT A STYLE. `aios_grid.rows_from_pool` puts `pid`, `_created`, + `lat` and `lon` on EVERY row regardless of what the view shows, and the stored row dict carries + every column the table has. Serialising a row and deleting the fields we do not want inverts + the failure: a column added next wave is INCLUDED by default and nobody notices, whereas an + allowlist that has not learned about it merely omits it. `routes_forms._public_form` is the + shipped precedent and says the same thing about itself. + """ + import core.user_tables as user_tables + from harness import filter_eval + + defn = user_tables.get(table_key, st=rt) or {} + fields = [f for f in (defn.get("fields") or []) if isinstance(f, dict)] + by_key = {str(f.get("key")): f for f in fields} + keys = _visible_keys(view, fields) + cfg = (view or {}).get("config") or {} + + # The row pool: the table's own rows, narrowed to the fields it declares, exactly as + # `routes_tables.scoped_pool` builds it for a materialised table. ⚠ A read-through table has + # no rows here and is refused at the MINT, so this branch is the only one that can be reached. + rows_src = [] + for rid, row in (defn.get("rows") or {}).items(): + if not str(rid).isdigit(): + continue + r = {k: v for k, v in (row or {}).items() if k in by_key} + r["pid"] = int(rid) + rows_src.append(r) + rows_src.sort(key=lambda r: r["pid"]) + + # The view's own row selection, through the SHARED evaluator. `_needs_a_reader` has already + # refused anything this context could not answer, so an empty result here means the filter + # genuinely matches nothing — not that we failed to resolve it. + ctx = filter_eval.EvalCtx(today=time.strftime("%Y-%m-%d")) + keep = set(filter_eval.visible_pids(cfg.get("filters"), rows_src, fields, ctx, + member_pids=cfg.get("memberPids"))) + chosen = [r for r in rows_src if r.get("pid") in keep] + + # ⛔⛔ COORDINATES RIDE A MAP WHEN THEY ARE COORDINATES — NOT WHEN THEY ARE VISIBLE, AND NOT + # BECAUSE OF WHAT A COLUMN IS CALLED. Two verifiers, one from each side, are why this reads + # the way it does; the first fix I wrote was wrong and the second report proved it. + # + # ⚠ THE LEAK (verifier #1, driven): a HIDDEN field keyed `lat` holding the string + # `CANARY-LAT-AAA` came out on the wire, because the pair was emitted before the `keys` + # projection and the bypass keyed on the FIELD NAME rather than on the value being a + # coordinate. So a column called `lat` could carry anything — a note, an address — and publish + # it. That is the real defect. + # + # ⛔ MY FIRST FIX GATED ON VISIBILITY, AND IT BROKE THE FEATURE (verifier #2): hiding the raw + # decimals is the NORMAL way somebody builds a Map view — nobody wants `38.7223` in the column + # list — so gating on `visible` meant an ordinary map published a page with no map, under two + # messages that contradicted each other ("no rows carry a location" vs "this view hides its + # location columns"). + # + # ⭐ THE RULE THAT SATISFIES BOTH: publishing a MAP is publishing WHERE THE ROWS ARE — that is + # what the sharer chose — so a real coordinate rides whether or not its column is shown, and a + # value that is not a coordinate never rides at all. `_coord` is the whole wall, and it is a + # VALUE test, so no naming convention can smuggle anything past it. + # ⚠ It mirrors `PublishedView.MapPlot`'s own `coord()` deliberately: the client must not plot + # what the server would not send, and the server must not send what the client would discard. + # Two normalizers on one question is a smell [[one-question-two-normalizers]] — kept here + # because they sit on opposite sides of a trust boundary, where the server's copy is the wall + # and the client's is display hygiene. + mode = _mode_of(view) + plotted = 0 + if mode == "map": + for r in chosen: + if _coord(r.get("lat"), 90) is not None and _coord(r.get("lon"), 180) is not None: + plotted += 1 + + limits = [] + if mode == "map" and chosen and not plotted: + # R6's second sentence. A map with nothing on it must say WHY — and this says the true + # why, which is about the DATA, because visibility is no longer part of the answer. + limits.append({ + "subject": "map", "effect": "not_plotted", + "detail": f"none of these {len(chosen)} rows carry a usable location", + "recommendation": "add `lat` and `lon` values to the records, then reload this link", + }) + if len(chosen) > PUBLIC_ROW_CAP: + # R6's second sentence. A short page that does not say it is short is the silent + # truncation the rule is actually about. + limits.append({ + "subject": "rows", "effect": "windowed", + "detail": f"this view has {len(chosen)} rows and a published page serves the first " + f"{PUBLIC_ROW_CAP}", + "recommendation": "narrow the view's filters, or share it with named people instead " + "of publishing a link", + }) + chosen = chosen[:PUBLIC_ROW_CAP] + + # ⛔ THE DISPLAY REFS ARE INTERSECTED WITH `visible`, NOT UNIONED INTO IT. A Map that colours + # by a column the view HIDES would otherwise put that column's value on every public row — + # the projection leak, arriving through the renderer rather than through the column list. The + # fail-closed choice is to drop the ref and render the map without colour; a published page + # that is slightly plainer beats one that ships a hidden column. + disp_in = (cfg.get("display") or {}) + display = {"mode": mode} + for ref in ("dateField", "stackField", "titleField", "colorField", "sizeField"): + if disp_in.get(ref) in keys: + display[ref] = disp_in[ref] + + return { + "title": str((view or {}).get("name") or "")[:200], + "mode": display["mode"], + "display": display, + "columns": [{"key": k, + "label": str(by_key[k].get("label") or k), + "type": str(by_key[k].get("type") or "text"), + **({"options": [str(o) for o in by_key[k]["options"]][:200]} + if isinstance(by_key[k].get("options"), list) and by_key[k].get("options") + else {})} + for k in keys], + # KEY BY KEY. `pid` rides because the client needs a stable row identity to render a list; + # it is a row NUMBER within this table and names nothing outside it. + # ⛔⛔ `lat`/`lon` RIDE ONLY ON A MAP **AND ONLY WHEN THE VIEW SHOWS THEM** — and the second + # half was missing, which was a LEAK. Found by a verifier that drove this route with a + # hidden field keyed `lat` carrying the string `CANARY-LAT-AAA`, and watched it come out + # on the wire. + # + # The first version emitted the pair BEFORE the `keys` projection, so `_visible_keys` never + # gated it. On a `ut_*` table coordinates are not magic: `routes_tables.scoped_pool` builds + # its row as `{k: v for k, v in row.items() if k in field_keys}`, so a value only survives + # if the table DECLARES a field keyed `lat`/`lon` — i.e. they are ORDINARY COLUMNS, and a + # view can hide them like any other. Hiding them therefore has to work here, because on a + # published page **the projection is the only wall there is** (`ut_*` databases have no + # hidden-field closure behind it, `routes_shares.py`'s docstring). + # + # ⚠ The bypass was keyed on the FIELD NAME, never on the value being a coordinate, so it + # forwarded whatever a column called `lat` happened to hold — a string, a note, anything. + # ⚠ And the shipped gate could not see it: it asserted the key NAMES rode on a map and not + # on a catalog, over a fixture whose rows carried no `lat` key at all — so it pinned the + # names while both values were `None` [[gate-answers-the-wrong-question]]. + # KEY BY KEY. `pid` rides because the client needs a stable row identity; it is a row + # NUMBER within this table and names nothing outside it. `lat`/`lon` ride only on a map, + # and only when they PARSE as coordinates — see the block above for why that is the test. + "rows": [{"pid": r.get("pid"), + **({"lat": _coord(r.get("lat"), 90), "lon": _coord(r.get("lon"), 180)} + if mode == "map" + and _coord(r.get("lat"), 90) is not None + and _coord(r.get("lon"), 180) is not None else {}), + **{k: r.get(k) for k in keys}} for r in chosen], + "total": len(chosen), + **({"limits": limits} if limits else {}), + } + + +async def _bounded_body(request: Request) -> dict: + """The request body, read WITH A BOUND — never `Body(...)`, never `await request.body()`. + + FastAPI reads and JSON-parses the WHOLE body before the handler's first line runs, so a + declared model is not a bound at all; `content-length` is caller-supplied, so checking it is + not one either. Streaming with a running total is the actual bound. + """ + size, chunks = 0, [] + async for chunk in request.stream(): + size += len(chunk) + if size > MAX_BODY_BYTES: + raise err(413, "body_too_large", "that request is too large") + chunks.append(chunk) + import json + try: + parsed = json.loads(b"".join(chunks) or b"{}") + except ValueError: + raise err(400, "bad_request", "that request could not be read") + return parsed if isinstance(parsed, dict) else {} + + +# ── THE SHARER'S DOOR (authenticated, creator-or-admin) ────────────────────────────────────── +# +# ⚠ THE NOUN IS `/view-link`, DELIBERATELY NOT `/views/{...}/publish`, and it mirrors the form +# door's `/form-link` for the same reason: a literal path segment sitting beside a `/{token}` +# wildcard is resolved by DECLARATION ORDER, and a noun that cannot collide with a token has no +# order to get wrong. + + +@router.get("/view-link") +def read_view_link(topic: str = "", view: str = "", + session: Session = Depends(require_session)): + """This view's link state. ⛔ A GET MUST NOT MINT — opening the panel is not publishing.""" + v, _mode = _may_administer_view(session, str(topic or ""), str(view or "")) + token, entry = _entry_for(session.runtime, str(topic), str(view)) + out = _state_of(token, entry) + # The presentational flags contract C1 put on the view spec, echoed back so the client can + # tell whether the two agree. They are a MIRROR of this bucket, never its source. + disp = ((v.get("config") or {}).get("display") or {}) if isinstance(v, dict) else {} + out["displayPublished"] = disp.get("published") is True + return out + + +@router.post("/view-link") +async def mint_view_link(request: Request, session: Session = Depends(require_session)): + """Publish this view, or change its access. `{topic, view, access?, passphrase?, rotate?}`. + + IDEMPOTENT WITHOUT `rotate`: opening the panel twice, or switching public↔password, must not + invalidate a link somebody already sent. `rotate: true` mints a fresh token and the previous + one dies in the SAME write, so there is never a window where both open the view. + """ + body = await _bounded_body(request) + topic, view = str(body.get("topic") or ""), str(body.get("view") or "") + _v, _mode = _may_administer_view(session, topic, view) + + access = "public" if body.get("access") == "public" else "password" + raw_pw = body.get("passphrase") + passphrase = "" if raw_pw is None else str(raw_pw) + if len(passphrase) > MAX_PASSPHRASE: + raise err(400, "passphrase_too_long", + f"a passphrase can be at most {MAX_PASSPHRASE} characters") + + existing_token, existing = _entry_for(session.runtime, topic, view) + rotate = bool(body.get("rotate")) + fresh = secrets.token_urlsafe(TOKEN_BYTES) + + # ⛔ A `password` LINK MUST END UP WITH A HASH, and there are exactly two ways to have one: + # the caller supplied a passphrase now, or one was already stored and is being kept. Anything + # else is refused HERE rather than stored and refused later — a link that cannot be opened by + # anybody is not a safe default, it is a broken feature that reads as a permission bug. + if access == "password": + if passphrase and len(passphrase) < MIN_PASSPHRASE: + raise err(400, "passphrase_too_short", + f"a passphrase needs at least {MIN_PASSPHRASE} characters") + if not passphrase and not (existing or {}).get("pw"): + raise err(400, "passphrase_required", + "a password-protected link needs a passphrase") + + minted = {"token": existing_token or fresh} + + def _set(cur): + cur = dict(cur or {}) + prev = None + for stored, where in list(cur.items()): + if (isinstance(where, dict) and where.get("table") == topic + and where.get("view") == view): + prev = dict(where) + cur.pop(stored, None) + if not rotate: + minted["token"] = str(stored) + if rotate or prev is None: + minted["token"] = fresh + entry = {"table": topic, "view": view, "access": access, + "createdBy": str((prev or {}).get("createdBy") or session.uname), + "createdAt": float((prev or {}).get("createdAt") or time.time())} + # ⛔ THE STORED PASSPHRASE SURVIVES A TRIP THROUGH `public`, and the first version DROPPED + # it — found by a verifier that traced the toggle rather than the happy path. Publishing + # as `public` skipped this block entirely, so the hash was destroyed while the TOKEN was + # kept; switching back to `password` then demanded a new passphrase, silently, while the + # panel's own sentence promised the opposite ("leave blank to keep the current one"). + # ⚠ Carrying it is inert, not lax: `_pw_ok` is consulted ONLY when `access == "password"` + # (`get_published`/`open_published` both test it first), and `_state_of` exposes a boolean, + # never the digest. A hash nobody can reach is not a secret in use — but a promise the UI + # makes and the store breaks is a defect either way, and the honest fix is to keep the + # promise rather than to reword it. + if passphrase: + salt = secrets.token_bytes(PW_SALT_BYTES) + entry["salt"] = salt.hex() + entry["iter"] = PW_ITERATIONS + entry["pw"] = _hash_pw(passphrase, salt) + elif (prev or {}).get("pw"): + # Carried key by key, so a future field on the entry is not silently inherited. + entry["salt"] = str((prev or {}).get("salt") or "") + entry["iter"] = int((prev or {}).get("iter") or PW_ITERATIONS) + entry["pw"] = str((prev or {}).get("pw") or "") + cur[minted["token"]] = entry + return cur + + session.runtime.update(TOKENS_KEY, _set, flush="sync") + _mirror_display(session.runtime, topic, view, True, access) + token, entry = _entry_for(session.runtime, topic, view) + # ⛔ NO FABRICATED FALLBACK ENTRY HERE. The first version answered + # `_state_of(token or minted["token"], entry or {"access": access, "pw": "x"})`, and that + # `"pw": "x"` would have reported `hasPassphrase: true` about a PUBLIC link if the read-back + # ever came back empty — a lie in the safe-looking direction, which is the kind that survives. + # If the write cannot be read back, say so; do not describe a state nobody verified. + if not entry: + raise err(503, "not_saved", + "the link was minted but could not be read back — reload and try again") + return _state_of(token, entry) + + +@router.delete("/view-link") +async def revoke_view_link(request: Request, session: Session = Depends(require_session)): + """Unpublish. `{topic, view}`. + + ⛔ REVOKE ROTATES — it does not merely unset a flag. The token STRING is dropped from the + index in this write, so the old link stops resolving immediately; and because a later publish + mints `secrets.token_urlsafe(24)` afresh, the revoked string can never come back. An + implementation that kept the token and flipped an `enabled` flag would leave the secret live + in the store, one bug away from working again, and would make "revoked" a property somebody + could forget to check on a code path added later. + """ + body = await _bounded_body(request) + topic, view = str(body.get("topic") or ""), str(body.get("view") or "") + _may_administer_view(session, topic, view) + + def _drop(cur): + cur = dict(cur or {}) + for stored, where in list(cur.items()): + if (isinstance(where, dict) and where.get("table") == topic + and where.get("view") == view): + cur.pop(stored, None) + return cur + + session.runtime.update(TOKENS_KEY, _drop, flush="sync") + # ⛔ AND THE MIRROR COMES DOWN IN THE SAME BREATH. Without this the grid goes on showing + # "published" about a link that no longer resolves — the half a reviewer caught. + _mirror_display(session.runtime, topic, view, False) + return {"published": False, "access": "password", "token": "", "url": ""} + + +# ── THE PUBLIC DOOR (no session — this is the whole feature) ───────────────────────────────── +# +# ⛔ NO `Depends(require_session)` ON EITHER ROUTE BELOW, DELIBERATELY. "Public" in this app is not +# a flag or an allow-list entry — `main.py` has no auth middleware and no exempt-path table; a +# route is public exactly by omitting the dependency. Which is why the two are kept together, +# under one banner, rather than filed beside the sharer's routes they resemble. + + +@router.get("/published/{token}") +def get_published(token: str, request: Request): + """The published view. Read-only, unauthenticated, projected to the view's own columns.""" + if not _rate_ok(_client_ip(request), time.time()): + raise err(429, "too_many_requests", "too many requests — wait a moment and try again") + found = _resolve(token) + if not found: + # ⛔⛔ W33-T70 — AN UNKNOWN TOKEN ANSWERS EXACTLY WHAT A LOCKED ONE ANSWERS. + # This used to `raise _refuse()`, and that 403 was a free oracle: one unauthenticated GET, + # no passphrase, no cost, told an enumerator whether a token was REAL. `_refuse`'s own + # docstring says the difference between "no such link" and "that link was revoked" must + # never be observable — and the route beside it published that difference in its status + # code. Sorting real tokens from fake ones is the whole of the work; once it is free, the + # passphrase is all that is left and it can be attacked offline-cheap. + # ⚠ SO THE UNKNOWN TOKEN GETS THE LOCKED SHAPE: `{"locked": true}` and nothing else — no + # title, no columns, no count, the same bytes a real password link returns before anyone + # has tried to open it. The guess then costs a POST with a passphrase, which is rate + # limited and PBKDF2-priced. A PUBLIC link still opens on this GET, which is what a public + # link is for; what stops being visible is which PASSWORD tokens exist. + _note_failure(_client_ip(request), time.time()) + _equalise_pw_cost(None) + return {"locked": True} + rt, _slug, table_key, view, entry = found + if entry.get("access") == "password": + # ⛔ THE SHAPE OF THE PASSWORD ANSWER, and it is not a refusal. A locked link must render + # a passphrase prompt, so this 200 says "there is something here and it is locked" and + # NOTHING else — no title, no column names, no row count. A 403 here would be + # indistinguishable from a bad token, which is right for a WRONG passphrase and wrong for + # a link the holder has not tried to open yet. + return {"locked": True} + return _public_view(rt, table_key, view) + + +@router.post("/published/{token}") +async def open_published(token: str, request: Request): + """Open a password-protected link. `{passphrase}`. + + ⛔ A WRONG PASSPHRASE AND A WRONG TOKEN ANSWER THE SAME 403, from the same `_refuse`. If they + differed, the route would confirm which tokens are real to anybody willing to send one guess — + and a 24-byte token's entire protection is that it cannot be found by guessing. + ⚠ The passphrase is compared against a PBKDF2 digest with `hmac.compare_digest`, and is never + stored, logged or echoed. D-130's scar: a form's invited-address list is IDENTIFICATION and + anyone may claim an identity; this is AUTHENTICATION and is treated as one. + """ + now = time.time() + if not _rate_ok(_client_ip(request), now): + raise err(429, "too_many_requests", "too many requests — wait a moment and try again") + body = await _bounded_body(request) + found = _resolve(token) + if not found: + # ⛔⛔ W33-T70 — SPEND THE PBKDF2 ANYWAY. A wrong token skipped the hash entirely and + # answered in ~0.4 ms while a wrong passphrase paid ~82 ms: a 206x gap, zero overlap in 24 + # samples, and a clean separation of real tokens from fake ones for anyone with a stopwatch. + # Both arms now cost the same, so the identical 403 above is finally identical in practice + # rather than only in wording. + _note_failure(_client_ip(request), now) + _equalise_pw_cost(None) + raise _refuse() + rt, _slug, table_key, view, entry = found + if entry.get("access") == "password" and not _pw_ok(entry, str(body.get("passphrase") or "")): + _note_failure(_client_ip(request), now) + raise _refuse() + # ⚠ A link with NO passphrase must still pay, or "this token is public" is readable from the + # clock on a route whose whole job is to be uninformative. + if entry.get("access") != "password": + _equalise_pw_cost(entry) + return _public_view(rt, table_key, view) diff --git a/api/routes_records.py b/api/routes_records.py index c37f74b261d63bb390db45660363e8d94ff796d4..ef0bb2c8153db1e40b2656a715f545d80b1fe22b 100644 --- a/api/routes_records.py +++ b/api/routes_records.py @@ -1,211 +1,211 @@ -"""Record detail routes: durable comments, scoped to the caller's book — ON EVERY DATABASE. - -⭐ WAVE 19 (owner item 12). This file used to be the CUSTOMER record's comment routes with a -customer-shaped wall bolted to the module import line: `_in_book` asked -`routes_customers.allowed_pids` whatever surface the browser was on. Opening a PRODUCT record and -typing a comment therefore asked the customer book about a CRC32 hash of a SKU code, and the -panel answered "that customer is not in your book" — the owner's report. The dangerous half is -the one nobody sees: a hash that collides with a real partner id passes the wall, and the comment -is filed against somebody's customer where the whole team can read it. - -THE SHAPE NOW: `?scope=` names the database (the same vocabulary `/workspace?scope=` and the -events route's `scopeKey` already speak), and `_pool_or_refuse` resolves BOTH halves of the wall -per scope — the GRANT and the ROW SET — by asking that topic's own route, never by re-deriving -one here: - - customer / cohort `routes_customers.allowed_pids` behind the `customer_data` grant - product `routes_products.scoped_pool` behind the `product_data` grant - ut_ `routes_tables.scoped_pids`, whose `_defn_or_refuse` IS the wall - (404 unknown / 403 not yours — a user table has no module grant). - ⭐ W33-T03/D-183: `scoped_pIDs`, not `scoped_pOOL` — the pool builds every - ROW to derive a pid set this module discards, and RAISES 409 on a - read-through grid past one window, which is why the record drawer painted - an error page on `ut_odoo_gl_lines`. - -⚠ THE PATH KEEPS ITS `/customers/` SEGMENT. It is the shipped URL and `verify_api.py`'s E1a -section pins it; the scope now travels beside it explicitly. A nicer noun is not worth churning -another session's gate mid-wave — the WALL is the query parameter, not the word. - -⚠ NO DEFAULT BEYOND THE LEGACY ONE. An absent `scope` means `customer`, which is what every -shipped client sent and what keeps the old callers byte-identical; an UNRECOGNISED scope is a -400, never a silent fallback to the customer book (`routes_grid._scope_or_400`'s rule, and for -the same reason: a typo served as `customer` answers a question nobody asked). -""" -from fastapi import APIRouter, Body, Depends, Query - -from deps import Session, err, require_session - -router = APIRouter(prefix="/api/v1") - -#: The customer topic's two names — one book, two surfaces (the Cohort page is the customer table -#: over hand-curated sets). Mirrors `modules.cohort.LEGACY_SCOPES` / `core.record_comments`. -_CUSTOMER_SCOPES = ("", "customer", "cohort") - - -def _scope_or_400(raw): - scope = str(raw or "customer").strip().lower() - if scope in _CUSTOMER_SCOPES or scope == "product" or scope.startswith("ut_"): - return "customer" if scope in _CUSTOMER_SCOPES else scope - raise err(400, "bad_scope", - "scope must be customer, cohort, product or a ut_ database — refusing to guess") - - -def _pool_or_refuse(session: Session, scope: str): - """The pids this session may attach comments to ON THIS DATABASE — grant wall included. - - Returns **`(pids, unbounded)`** — a 2-tuple on EVERY branch. Raises the topic's own 403/404/503, - so a caller who may not open the surface never learns anything about the row they asked about. - - ⛔ `unbounded` is TRUE only when the row set could not be ENUMERATED (a read-through grid past - one window), never when it is merely EMPTY. Those are opposite answers and `scoped_pids` returns - `frozenset()` for both — see the branch below. - ⚠ THE SHAPE IS A CONTRACT EVEN THOUGH THIS FUNCTION IS PRIVATE, and it has two consumers that - do not travel together: `_in_book` here, and a NEGATIVE CONTROL in `aios-web/api/verify_scopes.py` - that REPLACES this function with its own lambda. A gate's test double is a caller - ([[test-double-patched-by-a-name-list]]); when this signature moved, that double kept returning a - bare frozenset and the section died on `ValueError: too many values to unpack` — no tally, no - failing name. Change the shape here and that double changes with it. - """ - if scope == "product": - from routes_products import MODULE as PRODUCT_MODULE, scoped_pool - - session.require(PRODUCT_MODULE) - pids, _team, _rows, _fields = scoped_pool(session) - # ⚠ `(pids, unbounded)` on EVERY branch. This one returned a bare frozenset for ten minutes - # after the `ut_` branch grew its second element, and `_in_book`'s unpack would have raised - # a `TypeError` — a 500 on every product comment — while both other branches worked. A - # return shape is a contract even when the function is private. - return (pids, False) - if scope.startswith("ut_"): - # No module grant exists for a user table — `_defn_or_refuse` inside `scoped_pids` IS - # the wall (creator or admin, fail-closed), and it answers 404 before 403 exactly as the - # rows routes do. - # - # ⭐⭐ W33-T03 / D-183 — `scoped_pids`, NOT `scoped_pool`, AND THAT ONE WORD IS THE BUG. - # `scoped_pool` builds every ROW to derive a pid set this function then throws away, and on - # a read-through grid larger than one window it RAISES `409 window_required`. So opening the - # record drawer on `ut_odoo_gl_lines` (975,137 rows) painted an error page — for a panel - # that renders comments about ONE row it already has. `scoped_pids` answers the identical - # question (its docstring: *"the pid set is IDENTICAL, not merely equivalent"*) and takes - # W31-T20's `limits` OUT-PARAMETER instead of raising, which is the same shape `/workspace` - # used to become openable on those two grids. - from routes_tables import scoped_pids - - limits = [] - pids, _fields, _defn = scoped_pids(session, scope, limits=limits) - # ⛔ AN EMPTY PID SET AND AN UNRESOLVABLE ONE ARE OPPOSITE ANSWERS, and collapsing them is - # how a fail-closed default becomes a lie. `scoped_pids` returns `frozenset()` BOTH for a - # database with no rows and for a read-through grid too big to enumerate — it distinguishes - # them by APPENDING R6's sentence to `limits`. Without this branch the drawer would move - # from a 409 error page to a 403 "not in your book" on a row the user is looking at, which - # is the same defect wearing a politer message ([[empty-answer-vs-unfinished-answer]]). - # ⚠ ADMITTING HERE IS NOT A WIDENING, and the ruling is wave 27 / D-72: on a `ut_*` database - # THE TENANT IS THE UNIT — `scoped_pool` itself carries "no per-row owner filter; the - # table-level wall is the WHOLE wall". `_defn_or_refuse` has already run inside - # `scoped_pids` and answered 404/403. The pid set was only ever an existence check. - return (pids, bool(limits)) - from routes_customers import MODULE as CUSTOMER_MODULE, allowed_pids - - session.require(CUSTOMER_MODULE) - return (frozenset(allowed_pids(session)), False) - - -def _in_book(pid, session, scope): - pids, unbounded = _pool_or_refuse(session, scope) - if unbounded: - # The table wall passed and the row set is larger than this process will enumerate. Said - # out loud rather than silently admitting: R6's second sentence is that a limit which - # cannot be removed gets REPORTED, and this is the one place the report has no envelope to - # ride in. - print(f"[records] {scope}: pid membership unresolved (read-through beyond one window) — " - f"admitting on the table wall alone, per D-72") - return - if pid not in pids: - # 403, not 404: the record may exist, but this session may not inspect it. - raise err(403, "out_of_scope", "that record is not in your book") - - -def _unavailable(): - return err( - 503, - "store_unavailable", - "record comments are temporarily unavailable — no change was saved", - ) - - -# ⭐ WAVE 21 (D-17): the CANONICAL path is /records/{pid}/comments — comments hang off a RECORD -# in whatever topic `?scope=` names, and the customer-flavoured noun was wave-19 residue (the -# wall was always the query param). The old path stays as an ALIAS because the shipped client -# still calls it; verify_api pins the canonical path AND that the alias answers, so removing -# the alias later is a decision, never an accident. -@router.get("/records/{pid}/comments") -@router.get("/customers/{pid}/comments") -def comments(pid: int, scope: str = Query(default="customer"), - session: Session = Depends(require_session)): - from core import record_comments - - scope = _scope_or_400(scope) - _in_book(pid, session, scope) - try: - rows = record_comments.list_comments(session.runtime, pid, scope=scope) - except record_comments.CommentsUnavailable: - raise _unavailable() - return {"comments": rows} - - -@router.post("/records/{pid}/comments", status_code=201) -@router.post("/customers/{pid}/comments", status_code=201) -def create_comment( - pid: int, - body: dict = Body(default=None), - scope: str = Query(default="customer"), - session: Session = Depends(require_session), -): - from core import record_comments - - scope = _scope_or_400(scope) - _in_book(pid, session, scope) - try: - comment = record_comments.add_comment( - session.runtime, - pid, - (body or {}).get("body"), - session.uname, - session.user.get("name") or session.uname, - scope=scope, - ) - except ValueError as exc: - raise err(400, "bad_comment", str(exc)) - except record_comments.CommentsUnavailable: - raise _unavailable() - return {"comment": comment} - - -@router.delete("/records/{pid}/comments/{comment_id}") -@router.delete("/customers/{pid}/comments/{comment_id}") -def remove_comment( - pid: int, - comment_id: str, - scope: str = Query(default="customer"), - session: Session = Depends(require_session), -): - from core import record_comments - - scope = _scope_or_400(scope) - _in_book(pid, session, scope) - try: - deleted = record_comments.delete_comment( - session.runtime, - pid, - comment_id, - session.uname, - admin=session.admin, - scope=scope, - ) - except record_comments.CommentForbidden: - raise err(403, "comment_forbidden", "only the author may delete this comment") - except record_comments.CommentsUnavailable: - raise _unavailable() - if not deleted: - raise err(404, "comment_not_found", "that comment no longer exists") - return {"ok": True, "id": comment_id} +"""Record detail routes: durable comments, scoped to the caller's book — ON EVERY DATABASE. + +⭐ WAVE 19 (owner item 12). This file used to be the CUSTOMER record's comment routes with a +customer-shaped wall bolted to the module import line: `_in_book` asked +`routes_customers.allowed_pids` whatever surface the browser was on. Opening a PRODUCT record and +typing a comment therefore asked the customer book about a CRC32 hash of a SKU code, and the +panel answered "that customer is not in your book" — the owner's report. The dangerous half is +the one nobody sees: a hash that collides with a real partner id passes the wall, and the comment +is filed against somebody's customer where the whole team can read it. + +THE SHAPE NOW: `?scope=` names the database (the same vocabulary `/workspace?scope=` and the +events route's `scopeKey` already speak), and `_pool_or_refuse` resolves BOTH halves of the wall +per scope — the GRANT and the ROW SET — by asking that topic's own route, never by re-deriving +one here: + + customer / cohort `routes_customers.allowed_pids` behind the `customer_data` grant + product `routes_products.scoped_pool` behind the `product_data` grant + ut_ `routes_tables.scoped_pids`, whose `_defn_or_refuse` IS the wall + (404 unknown / 403 not yours — a user table has no module grant). + ⭐ W33-T03/D-183: `scoped_pIDs`, not `scoped_pOOL` — the pool builds every + ROW to derive a pid set this module discards, and RAISES 409 on a + read-through grid past one window, which is why the record drawer painted + an error page on `ut_odoo_gl_lines`. + +⚠ THE PATH KEEPS ITS `/customers/` SEGMENT. It is the shipped URL and `verify_api.py`'s E1a +section pins it; the scope now travels beside it explicitly. A nicer noun is not worth churning +another session's gate mid-wave — the WALL is the query parameter, not the word. + +⚠ NO DEFAULT BEYOND THE LEGACY ONE. An absent `scope` means `customer`, which is what every +shipped client sent and what keeps the old callers byte-identical; an UNRECOGNISED scope is a +400, never a silent fallback to the customer book (`routes_grid._scope_or_400`'s rule, and for +the same reason: a typo served as `customer` answers a question nobody asked). +""" +from fastapi import APIRouter, Body, Depends, Query + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + +#: The customer topic's two names — one book, two surfaces (the Cohort page is the customer table +#: over hand-curated sets). Mirrors `modules.cohort.LEGACY_SCOPES` / `core.record_comments`. +_CUSTOMER_SCOPES = ("", "customer", "cohort") + + +def _scope_or_400(raw): + scope = str(raw or "customer").strip().lower() + if scope in _CUSTOMER_SCOPES or scope == "product" or scope.startswith("ut_"): + return "customer" if scope in _CUSTOMER_SCOPES else scope + raise err(400, "bad_scope", + "scope must be customer, cohort, product or a ut_ database — refusing to guess") + + +def _pool_or_refuse(session: Session, scope: str): + """The pids this session may attach comments to ON THIS DATABASE — grant wall included. + + Returns **`(pids, unbounded)`** — a 2-tuple on EVERY branch. Raises the topic's own 403/404/503, + so a caller who may not open the surface never learns anything about the row they asked about. + + ⛔ `unbounded` is TRUE only when the row set could not be ENUMERATED (a read-through grid past + one window), never when it is merely EMPTY. Those are opposite answers and `scoped_pids` returns + `frozenset()` for both — see the branch below. + ⚠ THE SHAPE IS A CONTRACT EVEN THOUGH THIS FUNCTION IS PRIVATE, and it has two consumers that + do not travel together: `_in_book` here, and a NEGATIVE CONTROL in `aios-web/api/verify_scopes.py` + that REPLACES this function with its own lambda. A gate's test double is a caller + ([[test-double-patched-by-a-name-list]]); when this signature moved, that double kept returning a + bare frozenset and the section died on `ValueError: too many values to unpack` — no tally, no + failing name. Change the shape here and that double changes with it. + """ + if scope == "product": + from routes_products import MODULE as PRODUCT_MODULE, scoped_pool + + session.require(PRODUCT_MODULE) + pids, _team, _rows, _fields = scoped_pool(session) + # ⚠ `(pids, unbounded)` on EVERY branch. This one returned a bare frozenset for ten minutes + # after the `ut_` branch grew its second element, and `_in_book`'s unpack would have raised + # a `TypeError` — a 500 on every product comment — while both other branches worked. A + # return shape is a contract even when the function is private. + return (pids, False) + if scope.startswith("ut_"): + # No module grant exists for a user table — `_defn_or_refuse` inside `scoped_pids` IS + # the wall (creator or admin, fail-closed), and it answers 404 before 403 exactly as the + # rows routes do. + # + # ⭐⭐ W33-T03 / D-183 — `scoped_pids`, NOT `scoped_pool`, AND THAT ONE WORD IS THE BUG. + # `scoped_pool` builds every ROW to derive a pid set this function then throws away, and on + # a read-through grid larger than one window it RAISES `409 window_required`. So opening the + # record drawer on `ut_odoo_gl_lines` (975,137 rows) painted an error page — for a panel + # that renders comments about ONE row it already has. `scoped_pids` answers the identical + # question (its docstring: *"the pid set is IDENTICAL, not merely equivalent"*) and takes + # W31-T20's `limits` OUT-PARAMETER instead of raising, which is the same shape `/workspace` + # used to become openable on those two grids. + from routes_tables import scoped_pids + + limits = [] + pids, _fields, _defn = scoped_pids(session, scope, limits=limits) + # ⛔ AN EMPTY PID SET AND AN UNRESOLVABLE ONE ARE OPPOSITE ANSWERS, and collapsing them is + # how a fail-closed default becomes a lie. `scoped_pids` returns `frozenset()` BOTH for a + # database with no rows and for a read-through grid too big to enumerate — it distinguishes + # them by APPENDING R6's sentence to `limits`. Without this branch the drawer would move + # from a 409 error page to a 403 "not in your book" on a row the user is looking at, which + # is the same defect wearing a politer message ([[empty-answer-vs-unfinished-answer]]). + # ⚠ ADMITTING HERE IS NOT A WIDENING, and the ruling is wave 27 / D-72: on a `ut_*` database + # THE TENANT IS THE UNIT — `scoped_pool` itself carries "no per-row owner filter; the + # table-level wall is the WHOLE wall". `_defn_or_refuse` has already run inside + # `scoped_pids` and answered 404/403. The pid set was only ever an existence check. + return (pids, bool(limits)) + from routes_customers import MODULE as CUSTOMER_MODULE, allowed_pids + + session.require(CUSTOMER_MODULE) + return (frozenset(allowed_pids(session)), False) + + +def _in_book(pid, session, scope): + pids, unbounded = _pool_or_refuse(session, scope) + if unbounded: + # The table wall passed and the row set is larger than this process will enumerate. Said + # out loud rather than silently admitting: R6's second sentence is that a limit which + # cannot be removed gets REPORTED, and this is the one place the report has no envelope to + # ride in. + print(f"[records] {scope}: pid membership unresolved (read-through beyond one window) — " + f"admitting on the table wall alone, per D-72") + return + if pid not in pids: + # 403, not 404: the record may exist, but this session may not inspect it. + raise err(403, "out_of_scope", "that record is not in your book") + + +def _unavailable(): + return err( + 503, + "store_unavailable", + "record comments are temporarily unavailable — no change was saved", + ) + + +# ⭐ WAVE 21 (D-17): the CANONICAL path is /records/{pid}/comments — comments hang off a RECORD +# in whatever topic `?scope=` names, and the customer-flavoured noun was wave-19 residue (the +# wall was always the query param). The old path stays as an ALIAS because the shipped client +# still calls it; verify_api pins the canonical path AND that the alias answers, so removing +# the alias later is a decision, never an accident. +@router.get("/records/{pid}/comments") +@router.get("/customers/{pid}/comments") +def comments(pid: int, scope: str = Query(default="customer"), + session: Session = Depends(require_session)): + from core import record_comments + + scope = _scope_or_400(scope) + _in_book(pid, session, scope) + try: + rows = record_comments.list_comments(session.runtime, pid, scope=scope) + except record_comments.CommentsUnavailable: + raise _unavailable() + return {"comments": rows} + + +@router.post("/records/{pid}/comments", status_code=201) +@router.post("/customers/{pid}/comments", status_code=201) +def create_comment( + pid: int, + body: dict = Body(default=None), + scope: str = Query(default="customer"), + session: Session = Depends(require_session), +): + from core import record_comments + + scope = _scope_or_400(scope) + _in_book(pid, session, scope) + try: + comment = record_comments.add_comment( + session.runtime, + pid, + (body or {}).get("body"), + session.uname, + session.user.get("name") or session.uname, + scope=scope, + ) + except ValueError as exc: + raise err(400, "bad_comment", str(exc)) + except record_comments.CommentsUnavailable: + raise _unavailable() + return {"comment": comment} + + +@router.delete("/records/{pid}/comments/{comment_id}") +@router.delete("/customers/{pid}/comments/{comment_id}") +def remove_comment( + pid: int, + comment_id: str, + scope: str = Query(default="customer"), + session: Session = Depends(require_session), +): + from core import record_comments + + scope = _scope_or_400(scope) + _in_book(pid, session, scope) + try: + deleted = record_comments.delete_comment( + session.runtime, + pid, + comment_id, + session.uname, + admin=session.admin, + scope=scope, + ) + except record_comments.CommentForbidden: + raise err(403, "comment_forbidden", "only the author may delete this comment") + except record_comments.CommentsUnavailable: + raise _unavailable() + if not deleted: + raise err(404, "comment_not_found", "that comment no longer exists") + return {"ok": True, "id": comment_id} diff --git a/api/routes_script_views.py b/api/routes_script_views.py index 8db6fc5871a3e0b8f16bd0f077af8552c663cab6..a4feec967331ab1c46fa8a1d6a789eda61b6f63a 100644 --- a/api/routes_script_views.py +++ b/api/routes_script_views.py @@ -1,439 +1,439 @@ -"""routes_script_views.py — CONTRACT C3: a database View that is a PYTHON SCRIPT (R3 / R5 / R10). - -Owner item 6, verbatim (2026-08-18): *"Add code script as an interface (database View) so a user -can build whatever they want through the Agent chat interface. be able to create any dashboard -they want. User should have the ability to see the code AND the dashboard output of course… Limit -the code script View per database… Any agent can add into more AI script, so we can see different -versions or different things the AI code for us."* - - GET /api/v1/script-views?database=K the views bound to ONE database - POST /api/v1/script-views create one {database, name?, source} - GET /api/v1/script-views/{id} one view, its source and its history - PUT /api/v1/script-views/{id} a NEW VERSION of the source - DELETE /api/v1/script-views/{id} drop it - POST /api/v1/script-views/{id}/run run it -> {ok, spec | error, stdout, ms} - POST /api/v1/script-views/{id}/revert go back to an earlier version {version} - -⭐⭐ **R3 IS "SCOPED, NOT CAPPED", AND THE TWO HALVES POINT OPPOSITE WAYS.** *Scoped*: a view may -read ONLY the database it lives in, and a script that names another database is REFUSED with a -message naming both. *Not capped*: there is **no limit on how many script views a database may -carry**, because that is how an agent offers three attempts and the owner picks one. So nothing -below counts views. What IS bounded is what makes them big — one source is capped, and one view's -edit history is capped and REPORTS what it dropped. - -⛔ **THE RUN IS THE CALLER'S, NEVER THE AUTHOR'S (R5).** `script_sandbox.run_view` is handed -`session.user`, so a script written by an administrator and opened by a scoped analyst reads the -ANALYST's rows. The author decides what the code does; the reader decides what it can see. -⚠ And the reverse case is safe rather than lucky: a narrowly-scoped author cannot write a script -that exfiltrates anything, because the only thing a script can return is a render spec drawn on -the screen of the person who ran it. There is no network, no file and no second reader. - -⛔ **`run` IS `def`, NOT `async def`.** It waits on a subprocess for up to ten seconds; as a -coroutine that would block the event loop for every other request in the container. FastAPI runs a -plain `def` in the threadpool, which is what makes one slow script one slow REQUEST. -""" -import threading -from datetime import datetime, timezone - -from fastapi import APIRouter, Body, Depends - -from deps import Session, err, require_session - -router = APIRouter(prefix="/api/v1") - -#: The tenant's script views: `{id: record}`. Per tenant, so it rides `runtime.store_key`. -VIEWS_KEY = "script_views" - -MAX_NAME = 80 -MAX_SOURCE_BYTES = 128 * 1024 - -#: Edit history per view. ⚠ NOT a cap on the NUMBER of views (R3 forbids that) — a cap on how far -#: back ONE view's source is kept. Past this the oldest go and `trimmed` counts them, so a reader -#: can see the history is partial instead of concluding the view was only ever saved twice. -MAX_HISTORY = 40 - -#: ⛔ HOW MANY SCRIPTS MAY BE RUNNING IN THIS CONTAINER AT ONCE, and it is a REPORTED refusal -#: rather than a queue. Each run is a real subprocess with a ten-second wall clock; without this, -#: holding down refresh forks until the box gives up, and the tenant's ONE FastAPI process is what -#: gives up. A 429 that says so is honest; an unbounded fork is not. -MAX_CONCURRENT_RUNS = 4 -_RUN_SLOTS = threading.BoundedSemaphore(MAX_CONCURRENT_RUNS) - - -def _now(): - return datetime.now(timezone.utc).isoformat(timespec="seconds") - - -def _all(rt): - """`{id: record}` for one tenant. `{}` on any failure — an unreadable bucket must degrade to - "this database has no script views", never to a 500 on the view rail.""" - try: - found = rt.get(VIEWS_KEY) or {} - except Exception: # noqa: BLE001 - return {} - return found if isinstance(found, dict) else {} - - -def _database_ok(session, database): - """Does this database EXIST, and may this caller read it? Answered by C1, never by a list. - - ⛔ `perm_scope.may_read` ALONE IS NOT ENOUGH and the reason is easy to miss: it answers True - for an ADMIN on any key at all, including one no database answers to. So a create validated - with `may_read` would let an administrator bind a view to a typo and leave an orphan nothing - can ever run. `scoped_fields` is the cheap half of C1 (a `ut_*` definition, no rows) and it - RAISES `UnknownTable`, which is exactly the question being asked. - """ - import core.perm_scope as perm_scope - try: - perm_scope.scoped_fields(session.user, database, st=session.runtime) - except perm_scope.UnknownTable: - raise err(404, "no_database", f"there is no database '{database}' in this workspace") - except perm_scope.Denied: - raise err(403, "forbidden", f"your account may not read '{database}'") - except perm_scope.Unresolvable as exc: - # The database is real and cannot be served under this call's constraints. Standing rule - # 1's second sentence: report the cause and the recommendation, never a bare refusal. - raise err(409, "unresolvable", str(exc)) from None - - -def _clean_source(raw, *, allow_empty=False): - """The stored source, or a 400. `allow_empty` is CREATE's alone and the asymmetry is the point. - - ⭐⭐ W37-T41 — WHY CREATE MAY BE EMPTY AND SAVE MAY NOT. - Picking "Custom View" in the mode picker mints the view immediately, before a line of code - exists, so that `mode === 'script'` always implies a real id and no reader has to carry a - "the id might be missing" branch (the create semantics handed to lane C in mailbox E-7). - A view that has been created and not yet written is therefore a REAL, legible state: the editor - shows its starter placeholder and the Run control is right there. - A PUT is a different act. It appends a VERSION to a history capped at 40, and blanking a - working script by saving nothing over it is not an edit anybody means to make. So the guard - stays exactly where it was on that door, and a person who wants the view gone deletes it. - - ⛔ THIS WAS FOUND BY THE GATE, NOT BY READING. `core.script_sandbox.check_source("")` returns - None, so "an empty script is storable" looked true and was written into a mailbox answer another - lane was about to build on. The refusal was HERE, one layer above, in a function the sandbox - knows nothing about. Two validators for one question, disagreeing - [[one-question-two-normalizers]] — and the one that would have bitten a person is the one no - unit of this feature was asserting. - """ - source = str(raw or "") - if not source.strip() and not allow_empty: - raise err(400, "no_source", "a script view needs some code") - if len(source.encode("utf-8", "replace")) > MAX_SOURCE_BYTES: - raise err(413, "source_too_long", - f"a script view is at most {MAX_SOURCE_BYTES // 1024} KB of code") - return source - - -def _row(rec, *, source=False): - """One view as the list door reports it. ⚠ NO SOURCE unless asked: the rail lists names.""" - out = {"id": rec.get("id") or "", "database": rec.get("database") or "", - "name": rec.get("name") or "", "author": rec.get("author") or "", - "version": int(rec.get("version") or 1), - "created": rec.get("created") or "", "updated": rec.get("updated") or "", - "versions": len(rec.get("history") or []) + 1, - "trimmed": int(rec.get("trimmed") or 0), - # ⭐ W37-T46: which version this one was RESTORED from, when it was. Present on the - # row (not only the history) because the editor's header is where a person reads "what - # am I looking at", and "v5, restored from v2" is the sentence that makes a roll-back - # legible as an event rather than as a coincidence of matching code. - "restoredFrom": rec.get("restoredFrom")} - if source: - out["source"] = rec.get("source") or "" - out["history"] = [{"version": int(h.get("version") or 0), "author": h.get("author") or "", - "created": h.get("created") or "", - "bytes": len(str(h.get("source") or "").encode("utf-8", "replace"))} - for h in reversed(rec.get("history") or []) if isinstance(h, dict)] - return out - - -def _limits(): - return {"maxSourceBytes": MAX_SOURCE_BYTES, "maxName": MAX_NAME, - "maxHistory": MAX_HISTORY, "maxConcurrentRuns": MAX_CONCURRENT_RUNS, - # ⭐ SAID OUT LOUD, because R3's "not capped" half is the one a reader assumes wrong. - "maxViewsPerDatabase": None} - - -def _put(session, view_id, mutate): - """Read-modify-write ONE view, synchronously — the client re-reads the rail immediately.""" - def _set(cur): - cur = dict(cur or {}) - nxt = mutate(cur.get(view_id) if isinstance(cur.get(view_id), dict) else None) - if nxt is None: - cur.pop(view_id, None) - else: - cur[view_id] = nxt - return cur - - session.runtime.update(VIEWS_KEY, _set, flush="sync") - - -def _mine_or_admin(session, rec): - """Who may EDIT or DELETE a view: its author, or an administrator. - - ⚠ RUNNING IS A DIFFERENT QUESTION and deliberately wider — anybody who may read the database - may run any view on it, under their OWN scope. That is the whole of "so we can see different - versions or different things the AI code for us": a colleague's attempt is worth nothing if - only its author can open it. - """ - if session.admin or str(rec.get("author") or "") == session.uname: - return - raise err(403, "forbidden", "only the author or an administrator can change this script view") - - -# ── the routes ──────────────────────────────────────────────────────────────────────────────── -@router.get("/script-views") -def list_script_views(database: str = "", session: Session = Depends(require_session)): - """Every script view bound to ONE database, newest first. `database` is required.""" - key = str(database or "").strip() - if not key: - raise err(400, "no_database", "name the database whose script views you want") - _database_ok(session, key) - rows = [_row(rec) for rec in _all(session.runtime).values() - if isinstance(rec, dict) and str(rec.get("database") or "") == key] - rows.sort(key=lambda r: (r["created"], r["id"]), reverse=True) - return {"database": key, "views": rows, "limits": _limits()} - - -@router.post("/script-views") -def create_script_view(body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Create one. ⛔ THE SOURCE IS CHECKED BEFORE IT IS STORED, not first at run time. - - A script view that cannot be run is a broken feature the reader discovers by pressing a button, - and the agent that wrote it is long gone by then. `check_source` is pure and costs no process, - so the refusal arrives while the author still has the code in front of them. - """ - import secrets # noqa: PLC0415 - import core.script_sandbox as sandbox # noqa: PLC0415 - - body = body if isinstance(body, dict) else {} - database = str(body.get("database") or "").strip() - if not database: - raise err(400, "no_database", "a script view is bound to one database") - _database_ok(session, database) - # ⭐ `allow_empty` is CREATE's alone - see `_clean_source`. The picker mints a view the - # moment a person chooses the mode, so the empty source is the normal first state, not a slip. - source = _clean_source(body.get("source"), allow_empty=True) - refusal = sandbox.check_source(source) - if refusal is not None: - raise err(400, refusal.code, refusal.message) - - view_id = "sv_" + secrets.token_urlsafe(9) - rec = {"id": view_id, "database": database, - "name": " ".join(str(body.get("name") or "Script view").split())[:MAX_NAME], - "source": source, "author": session.uname, "version": 1, - "created": _now(), "updated": _now(), "history": [], "trimmed": 0} - _put(session, view_id, lambda _prior: rec) - fresh = _all(session.runtime).get(view_id) - if not isinstance(fresh, dict): - # The store took the write and did not record it. A 200 here would tell the author their - # script was saved when it was not. - raise err(503, "store_unavailable", "the script view was NOT created") - return {"view": _row(fresh, source=True), "limits": _limits()} - - -@router.get("/script-views/{view_id}") -def get_script_view(view_id: str, session: Session = Depends(require_session)): - """One view WITH its source and its edit history. Owner item 6's *"see the code"* half.""" - rec = _all(session.runtime).get(str(view_id)) - if not isinstance(rec, dict): - raise err(404, "no_view", "there is no script view with that id") - _database_ok(session, str(rec.get("database") or "")) - return {"view": _row(rec, source=True), "limits": _limits()} - - -@router.put("/script-views/{view_id}") -def update_script_view(view_id: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """A NEW VERSION of one view's source. The prior source is KEPT, never replaced in place.""" - import core.script_sandbox as sandbox # noqa: PLC0415 - - view_id = str(view_id) - rec = _all(session.runtime).get(view_id) - if not isinstance(rec, dict): - raise err(404, "no_view", "there is no script view with that id") - _mine_or_admin(session, rec) - body = body if isinstance(body, dict) else {} - source = _clean_source(body.get("source")) - refusal = sandbox.check_source(source) - if refusal is not None: - raise err(400, refusal.code, refusal.message) - - def _mutate(prior): - prior = dict(prior or rec) - history = list(prior.get("history") or []) - history.append({"version": int(prior.get("version") or 1), - "source": prior.get("source") or "", - "author": prior.get("author") or "", "created": prior.get("updated") or ""}) - dropped = max(0, len(history) - MAX_HISTORY) - prior["history"] = history[dropped:] if dropped else history - prior["trimmed"] = int(prior.get("trimmed") or 0) + dropped - prior["source"] = source - prior["version"] = int(prior.get("version") or 1) + 1 - prior["updated"] = _now() - if body.get("name"): - prior["name"] = " ".join(str(body["name"]).split())[:MAX_NAME] - return prior - - _put(session, view_id, _mutate) - fresh = _all(session.runtime).get(view_id) - if not isinstance(fresh, dict): - raise err(503, "store_unavailable", "the new version was NOT saved") - return {"view": _row(fresh, source=True), "limits": _limits()} - - -@router.post("/script-views/{view_id}/revert") -def revert_script_view(view_id: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Put one view back to an earlier version. D-338 / W37-T46. - - ⭐⭐ A ROLL-BACK IS A NEW VERSION, NEVER A DELETE OF THE ONES AFTER IT, and that is the whole - point of the ticket rather than an implementation detail. The history is what lets a person - trust an agent with their code: if reverting destroyed the versions it stepped over, one - mistaken revert would cost exactly what the history existed to protect, and there would be no - way back from the way back. So this appends, and `restoredFrom` records where the text came - from — the same posture the agent-harness roll-back already takes, so a reader who has seen one - is not surprised by the other. - - ⛔ IT LIVES ON THE SERVER BECAUSE THE HISTORY DOES. The list door deliberately serves history - entries WITHOUT their source (`_row`: the rail lists names), so a client cannot assemble an old - version's text to re-PUT it. Handing the source out just so the client could send it straight - back would widen a payload for a round trip that does not need to exist, and would make the - revert non-atomic: two calls, and a failure between them leaves the person looking at code that - is not what is stored. - ⚠ `_clean_source` is NOT re-run. The text being restored was already validated when it was - first saved, and a source that a later, stricter rule would now reject is exactly the source a - person is most likely to want back. `check_source` still runs, because the SANDBOX's refusal is - about what the code would DO and that must never be bypassed by a route. - """ - import core.script_sandbox as sandbox # noqa: PLC0415 - - view_id = str(view_id) - rec = _all(session.runtime).get(view_id) - if not isinstance(rec, dict): - raise err(404, "no_view", "there is no script view with that id") - _mine_or_admin(session, rec) - body = body if isinstance(body, dict) else {} - try: - want = int(body.get("version")) - except (TypeError, ValueError): - raise err(400, "no_version", "say which version to go back to") from None - - history = [h for h in (rec.get("history") or []) if isinstance(h, dict)] - match = next((h for h in history if int(h.get("version") or 0) == want), None) - if match is None: - # ⛔ TWO REASONS A VERSION IS MISSING AND THEY ARE NOT THE SAME FACT. One was never - # written; the other was TRIMMED off the 40-deep history and the record even counts how - # many. Saying "that version is gone" for the first would be a lie a person could act on. - trimmed = int(rec.get("trimmed") or 0) - if trimmed and want <= trimmed: - raise err(410, "version_trimmed", - f"version {want} is older than this view's history keeps " - f"({trimmed} earlier versions have been dropped)") - raise err(404, "no_version", f"this view has no version {want}") - - source = str(match.get("source") or "") - refusal = sandbox.check_source(source) - if refusal is not None: - # An older version that today's sandbox refuses. The person is told which version and why, - # rather than being handed a 400 about code they did not just type. - raise err(400, refusal.code, f"version {want} cannot be restored: {refusal.message}") - - def _mutate(prior): - prior = dict(prior or rec) - history_now = list(prior.get("history") or []) - history_now.append({"version": int(prior.get("version") or 1), - "source": prior.get("source") or "", - "author": prior.get("author") or "", - "created": prior.get("updated") or ""}) - dropped = max(0, len(history_now) - MAX_HISTORY) - prior["history"] = history_now[dropped:] if dropped else history_now - prior["trimmed"] = int(prior.get("trimmed") or 0) + dropped - prior["source"] = source - prior["version"] = int(prior.get("version") or 1) + 1 - prior["updated"] = _now() - # ⚠ ON THE RECORD, so the history reads as WHAT HAPPENED rather than as a version that - # mysteriously matches an older one. Without it a reader sees v5 and v2 with identical - # code and no way to tell a revert from a coincidence. - prior["restoredFrom"] = want - return prior - - _put(session, view_id, _mutate) - fresh = _all(session.runtime).get(view_id) - if not isinstance(fresh, dict): - raise err(503, "store_unavailable", "the roll-back was NOT saved") - return {"view": _row(fresh, source=True), "restoredFrom": want, "limits": _limits()} - - -@router.delete("/script-views/{view_id}") -def delete_script_view(view_id: str, session: Session = Depends(require_session)): - view_id = str(view_id) - rec = _all(session.runtime).get(view_id) - if not isinstance(rec, dict): - raise err(404, "no_view", "there is no script view with that id") - _mine_or_admin(session, rec) - _put(session, view_id, lambda _prior: None) - return {"deleted": view_id} - - -@router.post("/script-views/{view_id}/run") -def run_script_view(view_id: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """CONTRACT C3's run door: `{ok, spec | error, stdout, ms}`. - - ⛔ `spec` IS A DESCRIPTION THE CLIENT DRAWS. It is never HTML and never text the browser - executes — the sandbox refuses a spec carrying an `html`, `script`, `src`, `href` or `on*` key - before this function ever sees it, so a renderer cannot be talked into running something by a - script that was itself perfectly well behaved. - - ⛔ AND IT IS PLAIN `def`, NOT `async def` — see this module's header. A ten-second subprocess - wait on the event loop is a ten-second outage for the whole container. - """ - import core.script_sandbox as sandbox # noqa: PLC0415 - - rec = _all(session.runtime).get(str(view_id)) - if not isinstance(rec, dict): - raise err(404, "no_view", "there is no script view with that id") - database = str(rec.get("database") or "") - # A DRAFT run: the editor sends unsaved code so the author can try it before committing to it. - # It is checked exactly as a stored one is, because "unsaved" is not a permission. - draft = (body or {}).get("source") if isinstance(body, dict) else None - source = _clean_source(draft) if draft else str(rec.get("source") or "") - # ⚠ NO `_database_ok` CALL HERE. `run_view` asks C1 the identical question a line later, and - # asking twice builds a registry topic's pool twice. The three refusals are translated below - # instead, which is the same wall reached through the same door. - - if not _RUN_SLOTS.acquire(blocking=False): - raise err(429, "busy", - f"{MAX_CONCURRENT_RUNS} script views are already running on this server. " - f"Try again in a moment") - try: - out = sandbox.run_view(session.user, database, source, st=session.runtime) - finally: - _RUN_SLOTS.release() - - # ⛔ C1'S THREE REFUSALS ARE HTTP STATUSES, NOT `ok:false`. "You may not read this database" - # answered 200 would be a permission decision the client has to go looking for, and every - # other door in this app answers 403 for it. Everything BELOW this line is a well-formed - # request whose ANSWER is that the script did not produce a view — that is a 200 with - # `ok:false`, the shape `routes_web_agent.test` already uses for the same reason. - if out.get("code") == "unknown_table": - raise err(404, "no_database", out.get("message") or f"there is no database '{database}'") - if out.get("code") == "denied": - raise err(403, "forbidden", out.get("message") or "your account may not read that database") - if out.get("code") == "unresolvable": - refusal = err(409, "unresolvable", out.get("message") or "these rows cannot be served") - refusal.detail["error"]["limit"] = out.get("limit") or {} - raise refusal - - answer = {"ok": bool(out.get("ok")), "spec": out.get("spec"), - "stdout": out.get("stdout") or "", "truncated": bool(out.get("truncated")), - "ms": int(out.get("ms") or 0), "code": out.get("code") or "", - # ⭐ `caps` RIDES ON THE ANSWER (standing rule 1). On a POSIX host all three limits - # were applied; on a Windows host the memory and CPU ones were not, and a screen - # that claims an enforcement which did not happen is the failure the rule is about. - "caps": out.get("caps") or {}} - if not answer["ok"]: - answer["error"] = out.get("message") or "the script view did not produce a view" - return answer +"""routes_script_views.py — CONTRACT C3: a database View that is a PYTHON SCRIPT (R3 / R5 / R10). + +Owner item 6, verbatim (2026-08-18): *"Add code script as an interface (database View) so a user +can build whatever they want through the Agent chat interface. be able to create any dashboard +they want. User should have the ability to see the code AND the dashboard output of course… Limit +the code script View per database… Any agent can add into more AI script, so we can see different +versions or different things the AI code for us."* + + GET /api/v1/script-views?database=K the views bound to ONE database + POST /api/v1/script-views create one {database, name?, source} + GET /api/v1/script-views/{id} one view, its source and its history + PUT /api/v1/script-views/{id} a NEW VERSION of the source + DELETE /api/v1/script-views/{id} drop it + POST /api/v1/script-views/{id}/run run it -> {ok, spec | error, stdout, ms} + POST /api/v1/script-views/{id}/revert go back to an earlier version {version} + +⭐⭐ **R3 IS "SCOPED, NOT CAPPED", AND THE TWO HALVES POINT OPPOSITE WAYS.** *Scoped*: a view may +read ONLY the database it lives in, and a script that names another database is REFUSED with a +message naming both. *Not capped*: there is **no limit on how many script views a database may +carry**, because that is how an agent offers three attempts and the owner picks one. So nothing +below counts views. What IS bounded is what makes them big — one source is capped, and one view's +edit history is capped and REPORTS what it dropped. + +⛔ **THE RUN IS THE CALLER'S, NEVER THE AUTHOR'S (R5).** `script_sandbox.run_view` is handed +`session.user`, so a script written by an administrator and opened by a scoped analyst reads the +ANALYST's rows. The author decides what the code does; the reader decides what it can see. +⚠ And the reverse case is safe rather than lucky: a narrowly-scoped author cannot write a script +that exfiltrates anything, because the only thing a script can return is a render spec drawn on +the screen of the person who ran it. There is no network, no file and no second reader. + +⛔ **`run` IS `def`, NOT `async def`.** It waits on a subprocess for up to ten seconds; as a +coroutine that would block the event loop for every other request in the container. FastAPI runs a +plain `def` in the threadpool, which is what makes one slow script one slow REQUEST. +""" +import threading +from datetime import datetime, timezone + +from fastapi import APIRouter, Body, Depends + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + +#: The tenant's script views: `{id: record}`. Per tenant, so it rides `runtime.store_key`. +VIEWS_KEY = "script_views" + +MAX_NAME = 80 +MAX_SOURCE_BYTES = 128 * 1024 + +#: Edit history per view. ⚠ NOT a cap on the NUMBER of views (R3 forbids that) — a cap on how far +#: back ONE view's source is kept. Past this the oldest go and `trimmed` counts them, so a reader +#: can see the history is partial instead of concluding the view was only ever saved twice. +MAX_HISTORY = 40 + +#: ⛔ HOW MANY SCRIPTS MAY BE RUNNING IN THIS CONTAINER AT ONCE, and it is a REPORTED refusal +#: rather than a queue. Each run is a real subprocess with a ten-second wall clock; without this, +#: holding down refresh forks until the box gives up, and the tenant's ONE FastAPI process is what +#: gives up. A 429 that says so is honest; an unbounded fork is not. +MAX_CONCURRENT_RUNS = 4 +_RUN_SLOTS = threading.BoundedSemaphore(MAX_CONCURRENT_RUNS) + + +def _now(): + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +def _all(rt): + """`{id: record}` for one tenant. `{}` on any failure — an unreadable bucket must degrade to + "this database has no script views", never to a 500 on the view rail.""" + try: + found = rt.get(VIEWS_KEY) or {} + except Exception: # noqa: BLE001 + return {} + return found if isinstance(found, dict) else {} + + +def _database_ok(session, database): + """Does this database EXIST, and may this caller read it? Answered by C1, never by a list. + + ⛔ `perm_scope.may_read` ALONE IS NOT ENOUGH and the reason is easy to miss: it answers True + for an ADMIN on any key at all, including one no database answers to. So a create validated + with `may_read` would let an administrator bind a view to a typo and leave an orphan nothing + can ever run. `scoped_fields` is the cheap half of C1 (a `ut_*` definition, no rows) and it + RAISES `UnknownTable`, which is exactly the question being asked. + """ + import core.perm_scope as perm_scope + try: + perm_scope.scoped_fields(session.user, database, st=session.runtime) + except perm_scope.UnknownTable: + raise err(404, "no_database", f"there is no database '{database}' in this workspace") + except perm_scope.Denied: + raise err(403, "forbidden", f"your account may not read '{database}'") + except perm_scope.Unresolvable as exc: + # The database is real and cannot be served under this call's constraints. Standing rule + # 1's second sentence: report the cause and the recommendation, never a bare refusal. + raise err(409, "unresolvable", str(exc)) from None + + +def _clean_source(raw, *, allow_empty=False): + """The stored source, or a 400. `allow_empty` is CREATE's alone and the asymmetry is the point. + + ⭐⭐ W37-T41 — WHY CREATE MAY BE EMPTY AND SAVE MAY NOT. + Picking "Custom View" in the mode picker mints the view immediately, before a line of code + exists, so that `mode === 'script'` always implies a real id and no reader has to carry a + "the id might be missing" branch (the create semantics handed to lane C in mailbox E-7). + A view that has been created and not yet written is therefore a REAL, legible state: the editor + shows its starter placeholder and the Run control is right there. + A PUT is a different act. It appends a VERSION to a history capped at 40, and blanking a + working script by saving nothing over it is not an edit anybody means to make. So the guard + stays exactly where it was on that door, and a person who wants the view gone deletes it. + + ⛔ THIS WAS FOUND BY THE GATE, NOT BY READING. `core.script_sandbox.check_source("")` returns + None, so "an empty script is storable" looked true and was written into a mailbox answer another + lane was about to build on. The refusal was HERE, one layer above, in a function the sandbox + knows nothing about. Two validators for one question, disagreeing + [[one-question-two-normalizers]] — and the one that would have bitten a person is the one no + unit of this feature was asserting. + """ + source = str(raw or "") + if not source.strip() and not allow_empty: + raise err(400, "no_source", "a script view needs some code") + if len(source.encode("utf-8", "replace")) > MAX_SOURCE_BYTES: + raise err(413, "source_too_long", + f"a script view is at most {MAX_SOURCE_BYTES // 1024} KB of code") + return source + + +def _row(rec, *, source=False): + """One view as the list door reports it. ⚠ NO SOURCE unless asked: the rail lists names.""" + out = {"id": rec.get("id") or "", "database": rec.get("database") or "", + "name": rec.get("name") or "", "author": rec.get("author") or "", + "version": int(rec.get("version") or 1), + "created": rec.get("created") or "", "updated": rec.get("updated") or "", + "versions": len(rec.get("history") or []) + 1, + "trimmed": int(rec.get("trimmed") or 0), + # ⭐ W37-T46: which version this one was RESTORED from, when it was. Present on the + # row (not only the history) because the editor's header is where a person reads "what + # am I looking at", and "v5, restored from v2" is the sentence that makes a roll-back + # legible as an event rather than as a coincidence of matching code. + "restoredFrom": rec.get("restoredFrom")} + if source: + out["source"] = rec.get("source") or "" + out["history"] = [{"version": int(h.get("version") or 0), "author": h.get("author") or "", + "created": h.get("created") or "", + "bytes": len(str(h.get("source") or "").encode("utf-8", "replace"))} + for h in reversed(rec.get("history") or []) if isinstance(h, dict)] + return out + + +def _limits(): + return {"maxSourceBytes": MAX_SOURCE_BYTES, "maxName": MAX_NAME, + "maxHistory": MAX_HISTORY, "maxConcurrentRuns": MAX_CONCURRENT_RUNS, + # ⭐ SAID OUT LOUD, because R3's "not capped" half is the one a reader assumes wrong. + "maxViewsPerDatabase": None} + + +def _put(session, view_id, mutate): + """Read-modify-write ONE view, synchronously — the client re-reads the rail immediately.""" + def _set(cur): + cur = dict(cur or {}) + nxt = mutate(cur.get(view_id) if isinstance(cur.get(view_id), dict) else None) + if nxt is None: + cur.pop(view_id, None) + else: + cur[view_id] = nxt + return cur + + session.runtime.update(VIEWS_KEY, _set, flush="sync") + + +def _mine_or_admin(session, rec): + """Who may EDIT or DELETE a view: its author, or an administrator. + + ⚠ RUNNING IS A DIFFERENT QUESTION and deliberately wider — anybody who may read the database + may run any view on it, under their OWN scope. That is the whole of "so we can see different + versions or different things the AI code for us": a colleague's attempt is worth nothing if + only its author can open it. + """ + if session.admin or str(rec.get("author") or "") == session.uname: + return + raise err(403, "forbidden", "only the author or an administrator can change this script view") + + +# ── the routes ──────────────────────────────────────────────────────────────────────────────── +@router.get("/script-views") +def list_script_views(database: str = "", session: Session = Depends(require_session)): + """Every script view bound to ONE database, newest first. `database` is required.""" + key = str(database or "").strip() + if not key: + raise err(400, "no_database", "name the database whose script views you want") + _database_ok(session, key) + rows = [_row(rec) for rec in _all(session.runtime).values() + if isinstance(rec, dict) and str(rec.get("database") or "") == key] + rows.sort(key=lambda r: (r["created"], r["id"]), reverse=True) + return {"database": key, "views": rows, "limits": _limits()} + + +@router.post("/script-views") +def create_script_view(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Create one. ⛔ THE SOURCE IS CHECKED BEFORE IT IS STORED, not first at run time. + + A script view that cannot be run is a broken feature the reader discovers by pressing a button, + and the agent that wrote it is long gone by then. `check_source` is pure and costs no process, + so the refusal arrives while the author still has the code in front of them. + """ + import secrets # noqa: PLC0415 + import core.script_sandbox as sandbox # noqa: PLC0415 + + body = body if isinstance(body, dict) else {} + database = str(body.get("database") or "").strip() + if not database: + raise err(400, "no_database", "a script view is bound to one database") + _database_ok(session, database) + # ⭐ `allow_empty` is CREATE's alone - see `_clean_source`. The picker mints a view the + # moment a person chooses the mode, so the empty source is the normal first state, not a slip. + source = _clean_source(body.get("source"), allow_empty=True) + refusal = sandbox.check_source(source) + if refusal is not None: + raise err(400, refusal.code, refusal.message) + + view_id = "sv_" + secrets.token_urlsafe(9) + rec = {"id": view_id, "database": database, + "name": " ".join(str(body.get("name") or "Script view").split())[:MAX_NAME], + "source": source, "author": session.uname, "version": 1, + "created": _now(), "updated": _now(), "history": [], "trimmed": 0} + _put(session, view_id, lambda _prior: rec) + fresh = _all(session.runtime).get(view_id) + if not isinstance(fresh, dict): + # The store took the write and did not record it. A 200 here would tell the author their + # script was saved when it was not. + raise err(503, "store_unavailable", "the script view was NOT created") + return {"view": _row(fresh, source=True), "limits": _limits()} + + +@router.get("/script-views/{view_id}") +def get_script_view(view_id: str, session: Session = Depends(require_session)): + """One view WITH its source and its edit history. Owner item 6's *"see the code"* half.""" + rec = _all(session.runtime).get(str(view_id)) + if not isinstance(rec, dict): + raise err(404, "no_view", "there is no script view with that id") + _database_ok(session, str(rec.get("database") or "")) + return {"view": _row(rec, source=True), "limits": _limits()} + + +@router.put("/script-views/{view_id}") +def update_script_view(view_id: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """A NEW VERSION of one view's source. The prior source is KEPT, never replaced in place.""" + import core.script_sandbox as sandbox # noqa: PLC0415 + + view_id = str(view_id) + rec = _all(session.runtime).get(view_id) + if not isinstance(rec, dict): + raise err(404, "no_view", "there is no script view with that id") + _mine_or_admin(session, rec) + body = body if isinstance(body, dict) else {} + source = _clean_source(body.get("source")) + refusal = sandbox.check_source(source) + if refusal is not None: + raise err(400, refusal.code, refusal.message) + + def _mutate(prior): + prior = dict(prior or rec) + history = list(prior.get("history") or []) + history.append({"version": int(prior.get("version") or 1), + "source": prior.get("source") or "", + "author": prior.get("author") or "", "created": prior.get("updated") or ""}) + dropped = max(0, len(history) - MAX_HISTORY) + prior["history"] = history[dropped:] if dropped else history + prior["trimmed"] = int(prior.get("trimmed") or 0) + dropped + prior["source"] = source + prior["version"] = int(prior.get("version") or 1) + 1 + prior["updated"] = _now() + if body.get("name"): + prior["name"] = " ".join(str(body["name"]).split())[:MAX_NAME] + return prior + + _put(session, view_id, _mutate) + fresh = _all(session.runtime).get(view_id) + if not isinstance(fresh, dict): + raise err(503, "store_unavailable", "the new version was NOT saved") + return {"view": _row(fresh, source=True), "limits": _limits()} + + +@router.post("/script-views/{view_id}/revert") +def revert_script_view(view_id: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Put one view back to an earlier version. D-338 / W37-T46. + + ⭐⭐ A ROLL-BACK IS A NEW VERSION, NEVER A DELETE OF THE ONES AFTER IT, and that is the whole + point of the ticket rather than an implementation detail. The history is what lets a person + trust an agent with their code: if reverting destroyed the versions it stepped over, one + mistaken revert would cost exactly what the history existed to protect, and there would be no + way back from the way back. So this appends, and `restoredFrom` records where the text came + from — the same posture the agent-harness roll-back already takes, so a reader who has seen one + is not surprised by the other. + + ⛔ IT LIVES ON THE SERVER BECAUSE THE HISTORY DOES. The list door deliberately serves history + entries WITHOUT their source (`_row`: the rail lists names), so a client cannot assemble an old + version's text to re-PUT it. Handing the source out just so the client could send it straight + back would widen a payload for a round trip that does not need to exist, and would make the + revert non-atomic: two calls, and a failure between them leaves the person looking at code that + is not what is stored. + ⚠ `_clean_source` is NOT re-run. The text being restored was already validated when it was + first saved, and a source that a later, stricter rule would now reject is exactly the source a + person is most likely to want back. `check_source` still runs, because the SANDBOX's refusal is + about what the code would DO and that must never be bypassed by a route. + """ + import core.script_sandbox as sandbox # noqa: PLC0415 + + view_id = str(view_id) + rec = _all(session.runtime).get(view_id) + if not isinstance(rec, dict): + raise err(404, "no_view", "there is no script view with that id") + _mine_or_admin(session, rec) + body = body if isinstance(body, dict) else {} + try: + want = int(body.get("version")) + except (TypeError, ValueError): + raise err(400, "no_version", "say which version to go back to") from None + + history = [h for h in (rec.get("history") or []) if isinstance(h, dict)] + match = next((h for h in history if int(h.get("version") or 0) == want), None) + if match is None: + # ⛔ TWO REASONS A VERSION IS MISSING AND THEY ARE NOT THE SAME FACT. One was never + # written; the other was TRIMMED off the 40-deep history and the record even counts how + # many. Saying "that version is gone" for the first would be a lie a person could act on. + trimmed = int(rec.get("trimmed") or 0) + if trimmed and want <= trimmed: + raise err(410, "version_trimmed", + f"version {want} is older than this view's history keeps " + f"({trimmed} earlier versions have been dropped)") + raise err(404, "no_version", f"this view has no version {want}") + + source = str(match.get("source") or "") + refusal = sandbox.check_source(source) + if refusal is not None: + # An older version that today's sandbox refuses. The person is told which version and why, + # rather than being handed a 400 about code they did not just type. + raise err(400, refusal.code, f"version {want} cannot be restored: {refusal.message}") + + def _mutate(prior): + prior = dict(prior or rec) + history_now = list(prior.get("history") or []) + history_now.append({"version": int(prior.get("version") or 1), + "source": prior.get("source") or "", + "author": prior.get("author") or "", + "created": prior.get("updated") or ""}) + dropped = max(0, len(history_now) - MAX_HISTORY) + prior["history"] = history_now[dropped:] if dropped else history_now + prior["trimmed"] = int(prior.get("trimmed") or 0) + dropped + prior["source"] = source + prior["version"] = int(prior.get("version") or 1) + 1 + prior["updated"] = _now() + # ⚠ ON THE RECORD, so the history reads as WHAT HAPPENED rather than as a version that + # mysteriously matches an older one. Without it a reader sees v5 and v2 with identical + # code and no way to tell a revert from a coincidence. + prior["restoredFrom"] = want + return prior + + _put(session, view_id, _mutate) + fresh = _all(session.runtime).get(view_id) + if not isinstance(fresh, dict): + raise err(503, "store_unavailable", "the roll-back was NOT saved") + return {"view": _row(fresh, source=True), "restoredFrom": want, "limits": _limits()} + + +@router.delete("/script-views/{view_id}") +def delete_script_view(view_id: str, session: Session = Depends(require_session)): + view_id = str(view_id) + rec = _all(session.runtime).get(view_id) + if not isinstance(rec, dict): + raise err(404, "no_view", "there is no script view with that id") + _mine_or_admin(session, rec) + _put(session, view_id, lambda _prior: None) + return {"deleted": view_id} + + +@router.post("/script-views/{view_id}/run") +def run_script_view(view_id: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """CONTRACT C3's run door: `{ok, spec | error, stdout, ms}`. + + ⛔ `spec` IS A DESCRIPTION THE CLIENT DRAWS. It is never HTML and never text the browser + executes — the sandbox refuses a spec carrying an `html`, `script`, `src`, `href` or `on*` key + before this function ever sees it, so a renderer cannot be talked into running something by a + script that was itself perfectly well behaved. + + ⛔ AND IT IS PLAIN `def`, NOT `async def` — see this module's header. A ten-second subprocess + wait on the event loop is a ten-second outage for the whole container. + """ + import core.script_sandbox as sandbox # noqa: PLC0415 + + rec = _all(session.runtime).get(str(view_id)) + if not isinstance(rec, dict): + raise err(404, "no_view", "there is no script view with that id") + database = str(rec.get("database") or "") + # A DRAFT run: the editor sends unsaved code so the author can try it before committing to it. + # It is checked exactly as a stored one is, because "unsaved" is not a permission. + draft = (body or {}).get("source") if isinstance(body, dict) else None + source = _clean_source(draft) if draft else str(rec.get("source") or "") + # ⚠ NO `_database_ok` CALL HERE. `run_view` asks C1 the identical question a line later, and + # asking twice builds a registry topic's pool twice. The three refusals are translated below + # instead, which is the same wall reached through the same door. + + if not _RUN_SLOTS.acquire(blocking=False): + raise err(429, "busy", + f"{MAX_CONCURRENT_RUNS} script views are already running on this server. " + f"Try again in a moment") + try: + out = sandbox.run_view(session.user, database, source, st=session.runtime) + finally: + _RUN_SLOTS.release() + + # ⛔ C1'S THREE REFUSALS ARE HTTP STATUSES, NOT `ok:false`. "You may not read this database" + # answered 200 would be a permission decision the client has to go looking for, and every + # other door in this app answers 403 for it. Everything BELOW this line is a well-formed + # request whose ANSWER is that the script did not produce a view — that is a 200 with + # `ok:false`, the shape `routes_web_agent.test` already uses for the same reason. + if out.get("code") == "unknown_table": + raise err(404, "no_database", out.get("message") or f"there is no database '{database}'") + if out.get("code") == "denied": + raise err(403, "forbidden", out.get("message") or "your account may not read that database") + if out.get("code") == "unresolvable": + refusal = err(409, "unresolvable", out.get("message") or "these rows cannot be served") + refusal.detail["error"]["limit"] = out.get("limit") or {} + raise refusal + + answer = {"ok": bool(out.get("ok")), "spec": out.get("spec"), + "stdout": out.get("stdout") or "", "truncated": bool(out.get("truncated")), + "ms": int(out.get("ms") or 0), "code": out.get("code") or "", + # ⭐ `caps` RIDES ON THE ANSWER (standing rule 1). On a POSIX host all three limits + # were applied; on a Windows host the memory and CPU ones were not, and a screen + # that claims an enforcement which did not happen is the failure the rule is about. + "caps": out.get("caps") or {}} + if not answer["ok"]: + answer["error"] = out.get("message") or "the script view did not produce a view" + return answer diff --git a/api/routes_shares.py b/api/routes_shares.py index fc484be07ca8059fdf3c2c1227da64253250a5fd..d59c96b837a5a137b3f4e8b5ff43c9570e32ecf1 100644 --- a/api/routes_shares.py +++ b/api/routes_shares.py @@ -1,452 +1,458 @@ -"""routes_shares.py — the manage-access surface (wave 20, owner ruling R10, contract C-SHARE). - - GET /api/v1/share/{kind}/{oid} -> {owner, entries:[{user,role}], mayAdminister, people} - PUT /api/v1/share/{kind}/{oid} <- {entries:[{user,role}]} (REPLACES the set) - GET /api/v1/share/mine -> {view:[id], folder:[id], database:[id]} - -`kind` ∈ view | folder | database | field. Roles are `view` | `edit` — the same two words the -view rail already speaks, now extended to folders, databases and COLUMNS so there is ONE -vocabulary in the UI (R10: "the same picker views use"). - -⭐⭐ **W38-T16 — `field` IS THE FOURTH KIND, AND ITS `oid` IS TOPIC-QUALIFIED: `":"`** -(`shares.field_oid`). A bare column key repeats across databases — `notes` exists on a dozen — -so a grant stored under one would admit the grantee to every `notes` column in the tenant at -once. ⛔ THREE functions in this file branch on kind and ALL THREE need the new one, which is not -obvious because only two of them fail loudly: `_owns_object` (without it a column's own creator -is 404'd trying to share the thing they just made) and `_object_ref` (without it `route` is None, -`_notify_new_grantees` returns early, and the grantee is **granted and never told** — owner item -18's silent half, reopened one kind over). `_can_see_object` stays deliberately CLOSED for -anything that is not a view. - -⛔ **RE-SHARING IS THE OWNER'S, AND THAT IS ENFORCED HERE, NOT IN THE CLIENT.** `PUT` requires -`shares.may_administer` (owner or admin). A collaborator with `edit` may change an object's -CONTENT and may not change who else can reach it — otherwise anyone you shared a view with could -widen it to everyone, or grant themselves ownership and lock you out. The client greys the editor -for non-administrators; that is a courtesy, and this check is the wall. - -⚠ **THE GRANT NEVER WIDENS PAST THE MODULE WALL — ON A GOVERNED MODULE.** `*` ("everyone") means -every account that can already open the surface: `require_session` plus the topic's own gate run -first, and for `customer_data` / `product_data` the receiver's own row scope and hidden-field -closure run BEFORE any foreign view is merged. Sharing there can only narrow-or-equal the set that -could already reach the data ([[aios-permissioning]]). - -⛔⛔ **AND THAT SENTENCE IS FALSE FOR `kind='database'`, WHICH IS WHY IT NOW SAYS "ON A GOVERNED -MODULE" (W32-T26, audit S-8).** `routes_admin._PERM_MODULES` is `("customer_data","product_data")` -and `_clean_perms` **400s** on anything else, so **no row filter and no hidden field can even be -DECLARED for a `ut_*` database** — `routes_tables.py` makes zero `perm_scope` calls and passes -`hidden_keys=frozenset()`. There is no module wall behind a user table for a grant to be bounded -by: **this registry IS the wall.** So a `database` grant is ALL-OR-NOTHING — every row, every -column — and an `*` database grant admits every account in the tenant to all of it. -That is a real capability, deliberately kept; what was wrong was a docstring promising a second -wall that does not exist for this kind. Scoping user tables is booked, not done -(`waves/wave32/sharing-audit.md` S-8). - -⚠ **TWO SYSTEMS ANSWER "IS THIS SHARED", AND THEY ARE NOT THE SAME ONE (audit S-4).** THIS -registry decides who appears in *"Shared with me"* and who may re-share. **`table_store.is_shared` -— the view's own `permissions` — is what actually decides who may OPEN a view.** A grant here -whose object is invisible under that one is a row in a list that opens a refusal, which is what -made item 18 worth auditing. `_entries_or_400` closes the common cause (a name nobody has), but -the two vocabularies are still two. -""" -from fastapi import APIRouter, Body, Depends - -import core.shares as shares -import core.users as users -from deps import Session, err, require_session -# ⭐ W32-T28 (C3) — the SHARE notification's topic word, imported from the module that CLASSIFIES -# it (`routes_alerts.notification_view`) rather than typed again here. The producer and the -# reader agreeing about one string is the whole difference between an Inbox row that opens the -# shared database and one that is quietly unclickable. -from routes_alerts import SHARE_TOPIC as _SHARE_TOPIC - -router = APIRouter(prefix="/api/v1") - - -def _kind_or_400(raw): - try: - return shares._check_kind(raw) - except ValueError as e: - raise err(400, "bad_kind", str(e)) - - -# ── ⭐⭐ WAVE 32 · T26 (owner item 18, ruling R12) — THE WALL THIS FILE SAID IT HAD ───────────── -# -# `put_share`'s comment used to justify the first-claim rule with *"reaching this route at all -# means passing the surface's own wall"*. **There was no such wall.** `kind` and `oid` are free -# strings off the URL and the only dependency was `require_session`, so any signed-in account -# could `PUT` a grant on an id it had never seen. Because the 403 sat behind `if rec["owner"]`, -# an object with no grant record skipped the check entirely and the caller was stamped OWNER — -# sticky, so **the real creator was then refused on their own view, permanently.** Driven, not -# argued: `waves/wave32/sharing-audit.md` S-1 carries the four-step transcript. -# -# ⚠ AND IT WAS SILENT ON BOTH SIDES. The claimant does not even see the object in their own -# "Shared with me" (`shared_with` excludes what you own), so nothing appears anywhere until the -# victim next opens the dialog. - -#: The built-in grid topics. A view or folder lives in `{topic}_table_workspace`, and the share -#: route is not told which topic — so resolving one means asking each. -_BUILTIN_TOPICS = ("customer", "product") - - -def _topics(session): - """Every topic whose workspace could hold a view or folder for this tenant. - - ⚠ `all_defs`, never `all_tables` — the latter is the whole 28.6 MB row payload (~703 ms on - tenant #0) to answer a question about KEYS (D-185). - """ - try: - import core.user_tables as ut - return (*_BUILTIN_TOPICS, *(ut.all_defs(st=session.runtime) or {})) - except Exception: # noqa: BLE001 - return _BUILTIN_TOPICS - - -def _owns_object(session, kind, oid): - """May this caller CLAIM an object that has no grant record yet — i.e. do they own it? - - ⛔ THIS GUARDS THE CLAIM, NOT THE READ, AND THAT IS DELIBERATE. Resolving a view means asking - each topic's workspace in turn, which is N store reads; making every share call pay that - would put a loop on a route the manage-access dialog opens. The dangerous path is the one - where a caller is about to be stamped OWNER of something nobody owns — so the resolution runs - exactly there, and the common path (a record exists, `may_administer` decides) is untouched. - """ - if session.admin: - return True - if kind == "field": - # ⭐⭐ W38-T16 — A COLUMN'S OWNER IS ITS `createdBy`, WHICH THE CREATE DOOR ALREADY STAMPS - # (`routes_tables.patch_shared_cell`) and the DELETE door already reads as its wall (R8 / - # D-172: creator-or-admin). Read from the same place by all three, so a column cannot be - # deletable by one person and shareable by another. - # ⚠ THIS BRANCH IS NOT OPTIONAL AND ITS ABSENCE FAILS SILENTLY IN THE WORST DIRECTION: - # a brand-new column has no grant record, so `put_share` falls to this predicate — and - # without it the column's own creator is answered `404 no_object` on the first attempt to - # share the thing they just made. - table_key, field_key = shares.split_field_oid(oid) - if not table_key: - return False - try: - import core.shared_overlay as shared_overlay - defn = (shared_overlay.fields(table_key, st=session.runtime) or {}).get(field_key) - except Exception: # noqa: BLE001 - return False - return bool(defn) and str(defn.get("createdBy") or "") == str(session.uname) - if kind == "database": - # ⚠ `may_open` is THE resolver for a user table (its own docstring says so) and already - # admits creator, admin, or a `database` grantee. Re-implementing "who owns a table" - # here would be the second definition this wave keeps finding. - try: - import core.user_tables as ut - return bool(ut.may_open(oid, session.uname, is_admin=session.admin, - st=session.runtime)) - except Exception: # noqa: BLE001 - return False - try: - import core.table_store as table_store - except Exception: # noqa: BLE001 - return False - for topic in _topics(session): - try: - ops = table_store.make(f"{topic}_table_workspace", st=session.runtime) - hit = ops.find_view(oid) if kind == "view" else ops.find_folder(oid) - except Exception: # noqa: BLE001 - continue - if hit: - # `find_view`/`find_folder` answer `(owner_username, …)`. The claim belongs to the - # person whose personal stratum holds it — anybody else reaching this line is - # exactly the case S-1 describes. - return str(hit[0]) == str(session.uname) - return False - - -def _can_see_object(session, kind, oid): - """May this caller READ an object's grant list — i.e. can they reach the object at all? - - ⛔⛔ THIS IS DELIBERATELY WIDER THAN {@link _owns_object}, AND CONFLATING THE TWO IS A - REGRESSION I SHIPPED AND CAUGHT. The first version of T26 guarded BOTH doors with the - ownership test, which reads sensibly and is wrong for the read, because **`find_view` searches - PERSONAL STRATA ONLY** (its own docstring says so). So a view living in alice's stratum with - `permissions.edit = "collaborative"` and no grant record yet — a view bob **can open and edit - in the grid** — answered `404` when bob opened its manage-access dialog. Measured before - fixing: `table_store._may_see(view, "bob") is True` while `GET /share/view/vc` said - `404 no_object`. - ⚠ THAT IS THE AUDIT'S OWN S-4 BITING THE AUDIT'S OWN FIX: two systems answer "is this shared", - and the wall consulted the grant registry (system A) plus stratum ownership, never the view's - `permissions` (system B) — which is the one that actually decides who may OPEN it. - ⚠ And it hides the ANSWER, not just the editor. `ViewSidebar`'s Share row is deliberately not - gated on edit rights because *"hiding the row from everyone else would hide the ANSWER too — - 'who has this?' is a fair question for anyone the view was shared with"*. A 404 there tells a - legitimate collaborator their view does not exist. - - ⛔ THE CLAIM KEEPS THE NARROW TEST. Being able to SEE an object must not let you become its - owner — that is S-1, and widening this predicate onto `put_share` would re-open it. - """ - if _owns_object(session, kind, oid): - return True - if kind != "view": - # A folder carries no per-object visibility flag of its own, and a database's `may_open` - # (inside `_owns_object`) already admits grantees. Nothing wider to ask. - # ⭐ W38-T16 — AND `field` KEEPS THIS CLOSED, DELIBERATELY. A grantee never reaches here: - # `get_share` tests `role is None` first and a grant answers a role, so the only caller - # left is an account with no relationship to the column at all. Widening it would let any - # signed-in session enumerate who holds which column on a database they cannot open. - return False - try: - import core.table_store as table_store - for topic in _topics(session): - hit = table_store.make(f"{topic}_table_workspace", st=session.runtime).find_view(oid) - if hit: - return bool(table_store._may_see(hit[1] if len(hit) > 1 else {}, - session.uname, is_admin=session.admin)) - except Exception: # noqa: BLE001 - return False - return False - - -def _entries_or_400(session, entries): - """Validate a grant list against the tenant's REAL, ACTIVE accounts — and refuse BY NAME. - - ⛔ `core.shares._clean_entries` silently drops junk, and its docstring argues that correctly: - a UI mid-save must not lose the whole list to one malformed row. **But it validates the SHAPE - of a string and the role word — never that the user EXISTS, is ACTIVE, or is in this tenant**, - so a typo'd name is stored, reported as a successful save, and never reaches anybody. The - sharer believes the person has access. That is item 18's plain reading. - ⚠ The correct population is computed THREE FUNCTIONS BELOW and served to the picker - (`_people`). One route, two populations, and the write door was the permissive one. - ⚠ `*` (everyone) is not a user and is admitted deliberately — it is R10's vocabulary for - "every account that can already open the surface". - """ - known = {p["username"].strip().lower() for p in _people(session.tenant)} - unknown = [] - for e in entries or (): - if not isinstance(e, dict): - continue - user = str(e.get("user") or "").strip().lower() - if user and user != shares.EVERYONE and user not in known: - unknown.append(user) - if unknown: - raise err(400, "unknown_people", - "no active account in this workspace is named " - + ", ".join(sorted(set(unknown))) - + ". Nothing was shared. Pick people from the list rather than typing a name.") - - -@router.get("/share/mine") -def my_shares(session: Session = Depends(require_session)): - """Everything shared WITH me, by kind — the "Shared with me" rail section (R10). - - Registered before `/share/{kind}/{oid}` so the literal path wins the match; FastAPI resolves - in declaration order and `mine` would otherwise be read as a `kind`, answering 400 for a URL - that is not malformed at all. - """ - return shares.shared_with(session.uname, st=session.runtime) - - -@router.get("/share/{kind}/{oid}") -def get_share(kind: str, oid: str, session: Session = Depends(require_session)): - kind = _kind_or_400(kind) - rec = shares.grants(kind, oid, st=session.runtime) - role = shares.role_for(kind, oid, session.uname, is_admin=session.admin, st=session.runtime) - may_admin = shares.may_administer(kind, oid, session.uname, is_admin=session.admin, - st=session.runtime) - # ⭐ W32-T26 (audit S-3) — A STRANGER LEARNS NOTHING. This route used to answer for ANY id: - # who owns it, everyone it is granted to, and the tenant's whole username↔name directory — - # to any signed-in session, about objects it cannot open. Now a caller with no role on an - # object must prove they can reach it, and gets a 404 otherwise: the same answer a - # non-existent id gives, so the route cannot be used to probe which ids are real. - # ⚠ `role is None` is the cheap pre-test, so the N-topic resolution below runs only for a - # caller who has no relationship with the object at all. - if role is None and not _can_see_object(session, kind, oid): - raise err(404, "no_object", "no such item, or it is not shared with this account") - return { - **rec, - "role": role, - "mayAdminister": may_admin, - # ⚠ WAVE 21 (C1 identity fix): grant entries BIND on USERNAMES, so the picker must carry - # them. `assignable_people` serves bare display names because `user`-kind CELLS store - # display names — that list's shape cannot change without migrating cell values — so - # this route serves objects of its own. Existing grants that were written as lowercased - # display names are normalised by the wave-21 cleanup script. - # ⭐ W32-T26 (audit S-3) — the roster is the EDITOR's data, so it rides only for a caller - # who may open the editor. A read-only grantee gets the grant list (their fair question is - # "who else has this?") and not a directory of every account in the workspace. - "people": _people(session.tenant) if may_admin else [], - } - - -def _people(tenant): - """[{username, name}] for this tenant — same population as `assignable_people`, with the - BINDING identity alongside the display one.""" - try: - reg = users.registry() or {} - except Exception: - return [] - want = str(tenant or '').strip().lower() - out = [] - for uname, u in reg.items(): - if not isinstance(u, dict) or u.get('active') is False: - continue - if want and str(u.get('tenant') or 'royal-imports').strip().lower() != want: - continue - out.append({"username": str(uname), "name": str(u.get('name') or uname)}) - return sorted(out, key=lambda p: p["name"].lower()) - - -@router.put("/share/{kind}/{oid}") -def put_share(kind: str, oid: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - kind = _kind_or_400(kind) - body = body or {} - rec = shares.grants(kind, oid, st=session.runtime) - # An object with NO grant record yet has no owner — the first person to share it claims it. - # That is safe because reaching this route at all means passing the surface's own wall, and - # the alternative (refusing until somebody seeds an owner) would make a brand-new folder - # unshareable by the person who just made it. - if rec["owner"]: - if not shares.may_administer(kind, oid, session.uname, is_admin=session.admin, - st=session.runtime): - raise err(403, "not_owner", - "only the owner of this item (or an administrator) can change who it is " - "shared with") - # ⛔⛔ W32-T26 (audit S-1) — THE CLAIM NOW HAS A PRECONDITION. An object with no grant record - # is still claimed by the first person to share it — that rule is right, and refusing until - # somebody seeds an owner would make a brand-new folder unshareable by the person who just - # made it. What was missing is the half the old comment ASSERTED and the code never did: the - # claimant has to be able to reach the object. Without this, any signed-in account could - # stamp itself owner of an id it had never seen and lock the real creator out for good. - elif not _owns_object(session, kind, oid): - raise err(404, "no_object", "no such item, or it is not shared with this account") - entries = body.get("entries") - if not isinstance(entries, list): - raise err(400, "bad_entries", - "entries must be a list of {user, role}. Send [] to un-share, which is how " - "revoking is expressed") - _entries_or_400(session, entries) - out = shares.set_grants(kind, oid, entries, owner=rec["owner"] or session.uname, - st=session.runtime) - _notify_new_grantees(session, kind, oid, before=rec["entries"], after=out.get("entries") or []) - return out - - -def _notify_new_grantees(session, kind, oid, before, after): - """⭐⭐ W32-T28 (owner item 18's last clause, contract C3) — tell the RECEIVER, in their Inbox. - - Owner item 18 ends *"being shared a database notifies the receiver"*. Until now sharing was - silent: the grant landed in a rail section the receiver had to notice on their own, which is - why "I shared it with you" and "I never saw it" were both true. - - ⛔ WRITTEN ON THE SHARE, NEVER POLLED. `/notifications` re-evaluates view-ALERTS on read - because an alert is a live question about rows; a share is an EVENT that happened once, and - polling for it would mean re-deriving "was this new?" on every inbox open — the diff below - only exists here, at the moment the set changes. - - ⚠ ONLY THE NEWLY ADDED. `PUT` REPLACES the whole entry set (revoking is expressed by absence), - so every save re-sends everyone who was already there. Diffing against `before` is what stops - a rename or a role change from ringing the bell for people whose access did not change. - ⚠ `*` IS NOT NOTIFIED: there is no user to name, and minting one notification per account in - the tenant on a single click is a broadcast nobody asked for. The rail still shows it. - ⚠ IT NEVER RAISES. A notification that fails must not fail the share that triggered it — the - grant is the user's actual intent, and `core.alerts.notify` writes with `flush='async'`. - """ - try: - was = {e.get("user") for e in (before or ()) if isinstance(e, dict)} - fresh = [str(e.get("user")) for e in (after or ()) - if isinstance(e, dict) and e.get("user") not in was - and e.get("user") != shares.EVERYONE] - if not fresh: - return - import core.alerts as alerts - - label, route, view_id = _object_ref(session, kind, oid) - if not route: - # ⛔ NO ROUTE, NO NOTIFICATION — the receiver would get a row that opens nothing, and - # `notification_view` would have to invent a target. Silence is the honest answer - # here; the rail still shows the grant under "Shared with me". - return - sharer = str(session.user.get("name") or session.uname) - for user in fresh: - # ⚠ THE SHAPE IS `routes_alerts.notification_view`'s SHARE BRANCH, and the two must - # agree or the Inbox row is unclickable: `topic` selects the branch and `key` becomes - # `alertId`, which that branch reads as the id to open. Both constants are IMPORTED - # from there rather than typed again — one vocabulary, one owner. - # ⭐⭐ W33-T28 (`ASK C-14`, answered) — `actor` IS THE SENDER, AND IT IS THE ONLY WAY - # THE INBOX CAN NAME ONE. An alert and an automation have no person behind them and - # are honestly named by their machine; a SHARE has a real person, and only this call - # site knows who. ⛔ It is passed as its OWN field rather than recovered from the - # `detail` prose below: a sender parsed out of " shared this with you" breaks - # the first time the sentence is reworded, silently, in the header - # [[grep-output-is-not-source]]. The prose stays as the body; this is the From. - alerts.notify(user, label, topic=_SHARE_TOPIC, key=route, row_id=view_id, - detail=f"{sharer} shared this with you", actor=sharer, - st=session.runtime) - except Exception: # noqa: BLE001 - return - - -def _object_ref(session, kind, oid): - """`(label, route, view_id)` — what to CALL the shared thing, and where it OPENS. - - ⛔ THE ROUTE IS RESOLVED HERE, NOT SHAPED IN THE CONSUMER, AND THE FIRST VERSION GOT IT - WRONG: it put the raw `oid` in the notification's key, so a shared VIEW produced - `target: {module: "database", id: "view_42"}` — an instruction to open a database named - `view_42`. It read perfectly in the payload and would have opened nothing. **A view is not - addressable on its own; it is a SELECTION inside a topic's grid**, so the pair is what has to - travel. Caught by looking at the notification the driver actually produced, not by reading - the code back. - - ⚠ `label` never falls back to a raw id. A notification headed `ut_leads_3f2a` tells the - receiver nothing they can act on, and the id is already in the target. - ⚠ An unresolvable object answers `route=None`, and the caller then sends NOTHING rather than - a row that opens nowhere. - """ - try: - if kind == "field": - # ⭐⭐ W38-T16 — A COLUMN IS NOT ADDRESSABLE ON ITS OWN, exactly as a view is not: it - # is a column INSIDE a database, so the target that travels is the DATABASE. Without - # this branch the function falls through to the view/folder loop, finds nothing, - # answers `route=None` — and `_notify_new_grantees` returns EARLY. The grant lands and - # the receiver is never told, which is the silent half of owner item 18 reopened one - # kind over. - from routes_alerts import route_for_topic - table_key, field_key = shares.split_field_oid(oid) - if not table_key: - return ("A column", None, "") - try: - import core.shared_overlay as shared_overlay - defn = (shared_overlay.fields(table_key, st=session.runtime) or {}).get(field_key) - except Exception: # noqa: BLE001 - defn = None - label = str((defn or {}).get("label") or "").strip() or field_key - # ⚠ TWO SPELLINGS REACH THIS LINE AND ONE MAP ANSWERS BOTH. `shared_overlay` is keyed - # by whatever the calling door already held: a `ut_*` database uses its bare key, - # while a registry topic uses `_table_workspace` (`product_data.TABLE_KEY`). - # `route_for_topic` speaks the GRID SCOPE vocabulary (`customer`, not - # `customer_data`), so the suffix comes off before it is asked — rather than a second - # route table being written here, which is how the two come apart. - _WS = "_table_workspace" - scope = table_key[:-len(_WS)] if table_key.endswith(_WS) else table_key - return (label, route_for_topic(scope) or None, "") - if kind == "database": - import core.user_tables as ut - defn = (ut.all_defs(st=session.runtime) or {}).get(str(oid)) or {} - # A user table IS its own route key in both vocabularies (`route_for_topic`). - return (str(defn.get("label") or "").strip() or "A database", str(oid), "") - import core.table_store as table_store - from routes_alerts import route_for_topic - for topic in _topics(session): - ops = table_store.make(f"{topic}_table_workspace", st=session.runtime) - hit = ops.find_view(oid) if kind == "view" else ops.find_folder(oid) - if not hit: - continue - route = route_for_topic(topic) - if not route: - break - row = hit[1] if len(hit) > 1 else {} - name = str((row or {}).get("name") or "").strip() - # ⚠ Only a VIEW carries a selection. A folder is a rail grouping, so the target opens - # the grid and stops there rather than naming a view the receiver did not get. - return (name or ("A view" if kind == "view" else "A folder"), - route, str(oid) if kind == "view" else "") - except Exception: # noqa: BLE001 - pass - return ({"view": "A view", "folder": "A folder", - "field": "A column"}.get(kind, "An item"), None, "") +"""routes_shares.py — the manage-access surface (wave 20, owner ruling R10, contract C-SHARE). + + GET /api/v1/share/{kind}/{oid} -> {owner, entries:[{user,role}], mayAdminister, people} + PUT /api/v1/share/{kind}/{oid} <- {entries:[{user,role}]} (REPLACES the set) + GET /api/v1/share/mine -> {view:[id], folder:[id], database:[id]} + +`kind` ∈ view | folder | database | field. Roles are `view` | `edit` — the same two words the +view rail already speaks, now extended to folders, databases and COLUMNS so there is ONE +vocabulary in the UI (R10: "the same picker views use"). + +⭐⭐ **W38-T16 — `field` IS THE FOURTH KIND, AND ITS `oid` IS TOPIC-QUALIFIED: `":"`** +(`shares.field_oid`). A bare column key repeats across databases — `notes` exists on a dozen — +so a grant stored under one would admit the grantee to every `notes` column in the tenant at +once. ⛔ THREE functions in this file branch on kind and ALL THREE need the new one, which is not +obvious because only two of them fail loudly: `_owns_object` (without it a column's own creator +is 404'd trying to share the thing they just made) and `_object_ref` (without it `route` is None, +`_notify_new_grantees` returns early, and the grantee is **granted and never told** — owner item +18's silent half, reopened one kind over). `_can_see_object` stays deliberately CLOSED for +anything that is not a view. + +⛔ **RE-SHARING IS THE OWNER'S, AND THAT IS ENFORCED HERE, NOT IN THE CLIENT.** `PUT` requires +`shares.may_administer` (owner or admin). A collaborator with `edit` may change an object's +CONTENT and may not change who else can reach it — otherwise anyone you shared a view with could +widen it to everyone, or grant themselves ownership and lock you out. The client greys the editor +for non-administrators; that is a courtesy, and this check is the wall. + +⚠ **THE GRANT NEVER WIDENS PAST THE MODULE WALL — ON A GOVERNED MODULE.** `*` ("everyone") means +every account that can already open the surface: `require_session` plus the topic's own gate run +first, and for `customer_data` / `product_data` the receiver's own row scope and hidden-field +closure run BEFORE any foreign view is merged. Sharing there can only narrow-or-equal the set that +could already reach the data ([[aios-permissioning]]). + +⛔⛔ **AND THAT SENTENCE IS FALSE FOR `kind='database'`, WHICH IS WHY IT NOW SAYS "ON A GOVERNED +MODULE" (W32-T26, audit S-8).** `routes_admin._PERM_MODULES` is `("customer_data","product_data")` +and `_clean_perms` **400s** on anything else, so **no row filter and no hidden field can even be +DECLARED for a `ut_*` database** — `routes_tables.py` makes zero `perm_scope` calls and passes +`hidden_keys=frozenset()`. There is no module wall behind a user table for a grant to be bounded +by: **this registry IS the wall.** So a `database` grant is ALL-OR-NOTHING — every row, every +column — and an `*` database grant admits every account in the tenant to all of it. +That is a real capability, deliberately kept; what was wrong was a docstring promising a second +wall that does not exist for this kind. Scoping user tables is booked, not done +(`waves/wave32/sharing-audit.md` S-8). + +⚠ **TWO SYSTEMS ANSWER "IS THIS SHARED", AND THEY ARE NOT THE SAME ONE (audit S-4).** THIS +registry decides who appears in *"Shared with me"* and who may re-share. **`table_store.is_shared` +— the view's own `permissions` — is what actually decides who may OPEN a view.** A grant here +whose object is invisible under that one is a row in a list that opens a refusal, which is what +made item 18 worth auditing. `_entries_or_400` closes the common cause (a name nobody has), but +the two vocabularies are still two. +""" +from fastapi import APIRouter, Body, Depends + +import core.shares as shares +import core.users as users +from deps import Session, err, require_session +# ⭐ W32-T28 (C3) — the SHARE notification's topic word, imported from the module that CLASSIFIES +# it (`routes_alerts.notification_view`) rather than typed again here. The producer and the +# reader agreeing about one string is the whole difference between an Inbox row that opens the +# shared database and one that is quietly unclickable. +from routes_alerts import SHARE_TOPIC as _SHARE_TOPIC + +router = APIRouter(prefix="/api/v1") + + +def _kind_or_400(raw): + try: + return shares._check_kind(raw) + except ValueError as e: + raise err(400, "bad_kind", str(e)) + + +# ── ⭐⭐ WAVE 32 · T26 (owner item 18, ruling R12) — THE WALL THIS FILE SAID IT HAD ───────────── +# +# `put_share`'s comment used to justify the first-claim rule with *"reaching this route at all +# means passing the surface's own wall"*. **There was no such wall.** `kind` and `oid` are free +# strings off the URL and the only dependency was `require_session`, so any signed-in account +# could `PUT` a grant on an id it had never seen. Because the 403 sat behind `if rec["owner"]`, +# an object with no grant record skipped the check entirely and the caller was stamped OWNER — +# sticky, so **the real creator was then refused on their own view, permanently.** Driven, not +# argued: `waves/wave32/sharing-audit.md` S-1 carries the four-step transcript. +# +# ⚠ AND IT WAS SILENT ON BOTH SIDES. The claimant does not even see the object in their own +# "Shared with me" (`shared_with` excludes what you own), so nothing appears anywhere until the +# victim next opens the dialog. + +#: The built-in grid topics. A view or folder lives in `{topic}_table_workspace`, and the share +#: route is not told which topic — so resolving one means asking each. +_BUILTIN_TOPICS = ("customer", "product") + + +def _topics(session): + """Every topic whose workspace could hold a view or folder for this tenant. + + ⚠ `all_defs`, never `all_tables` — the latter is the whole 28.6 MB row payload (~703 ms on + tenant #0) to answer a question about KEYS (D-185). + """ + try: + import core.user_tables as ut + return (*_BUILTIN_TOPICS, *(ut.all_defs(st=session.runtime) or {})) + except Exception: # noqa: BLE001 + return _BUILTIN_TOPICS + + +def _owns_object(session, kind, oid): + """May this caller CLAIM an object that has no grant record yet — i.e. do they own it? + + ⛔ THIS GUARDS THE CLAIM, NOT THE READ, AND THAT IS DELIBERATE. Resolving a view means asking + each topic's workspace in turn, which is N store reads; making every share call pay that + would put a loop on a route the manage-access dialog opens. The dangerous path is the one + where a caller is about to be stamped OWNER of something nobody owns — so the resolution runs + exactly there, and the common path (a record exists, `may_administer` decides) is untouched. + """ + if session.admin: + return True + if kind == "field": + # ⭐⭐ W38-T16 — A COLUMN'S OWNER IS ITS `createdBy`, WHICH THE CREATE DOOR ALREADY STAMPS + # (`routes_tables.patch_shared_cell`) and the DELETE door already reads as its wall (R8 / + # D-172: creator-or-admin). Read from the same place by all three, so a column cannot be + # deletable by one person and shareable by another. + # ⚠ THIS BRANCH IS NOT OPTIONAL AND ITS ABSENCE FAILS SILENTLY IN THE WORST DIRECTION: + # a brand-new column has no grant record, so `put_share` falls to this predicate — and + # without it the column's own creator is answered `404 no_object` on the first attempt to + # share the thing they just made. + table_key, field_key = shares.split_field_oid(oid) + if not table_key: + return False + try: + import core.shared_overlay as shared_overlay + defn = (shared_overlay.fields(table_key, st=session.runtime) or {}).get(field_key) + except Exception: # noqa: BLE001 + return False + return bool(defn) and str(defn.get("createdBy") or "") == str(session.uname) + if kind == "database": + # ⚠ `may_open` is THE resolver for a user table (its own docstring says so) and already + # admits creator, admin, or a `database` grantee. Re-implementing "who owns a table" + # here would be the second definition this wave keeps finding. + try: + import core.user_tables as ut + return bool(ut.may_open(oid, session.uname, is_admin=session.admin, + st=session.runtime)) + except Exception: # noqa: BLE001 + return False + try: + import core.table_store as table_store + except Exception: # noqa: BLE001 + return False + for topic in _topics(session): + try: + ops = table_store.make(f"{topic}_table_workspace", st=session.runtime) + hit = ops.find_view(oid) if kind == "view" else ops.find_folder(oid) + except Exception: # noqa: BLE001 + continue + if hit: + # `find_view`/`find_folder` answer `(owner_username, …)`. The claim belongs to the + # person whose personal stratum holds it — anybody else reaching this line is + # exactly the case S-1 describes. + return str(hit[0]) == str(session.uname) + return False + + +def _can_see_object(session, kind, oid): + """May this caller READ an object's grant list — i.e. can they reach the object at all? + + ⛔⛔ THIS IS DELIBERATELY WIDER THAN {@link _owns_object}, AND CONFLATING THE TWO IS A + REGRESSION I SHIPPED AND CAUGHT. The first version of T26 guarded BOTH doors with the + ownership test, which reads sensibly and is wrong for the read, because **`find_view` searches + PERSONAL STRATA ONLY** (its own docstring says so). So a view living in alice's stratum with + `permissions.edit = "collaborative"` and no grant record yet — a view bob **can open and edit + in the grid** — answered `404` when bob opened its manage-access dialog. Measured before + fixing: `table_store._may_see(view, "bob") is True` while `GET /share/view/vc` said + `404 no_object`. + ⚠ THAT IS THE AUDIT'S OWN S-4 BITING THE AUDIT'S OWN FIX: two systems answer "is this shared", + and the wall consulted the grant registry (system A) plus stratum ownership, never the view's + `permissions` (system B) — which is the one that actually decides who may OPEN it. + ⚠ And it hides the ANSWER, not just the editor. `ViewSidebar`'s Share row is deliberately not + gated on edit rights because *"hiding the row from everyone else would hide the ANSWER too — + 'who has this?' is a fair question for anyone the view was shared with"*. A 404 there tells a + legitimate collaborator their view does not exist. + + ⛔ THE CLAIM KEEPS THE NARROW TEST. Being able to SEE an object must not let you become its + owner — that is S-1, and widening this predicate onto `put_share` would re-open it. + """ + if _owns_object(session, kind, oid): + return True + if kind != "view": + # A folder carries no per-object visibility flag of its own, and a database's `may_open` + # (inside `_owns_object`) already admits grantees. Nothing wider to ask. + # ⭐ W38-T16 — AND `field` KEEPS THIS CLOSED, DELIBERATELY. A grantee never reaches here: + # `get_share` tests `role is None` first and a grant answers a role, so the only caller + # left is an account with no relationship to the column at all. Widening it would let any + # signed-in session enumerate who holds which column on a database they cannot open. + return False + try: + import core.table_store as table_store + for topic in _topics(session): + hit = table_store.make(f"{topic}_table_workspace", st=session.runtime).find_view(oid) + if hit: + return bool(table_store._may_see(hit[1] if len(hit) > 1 else {}, + session.uname, is_admin=session.admin)) + except Exception: # noqa: BLE001 + return False + return False + + +def _entries_or_400(session, entries): + """Validate a grant list against the tenant's REAL, ACTIVE accounts — and refuse BY NAME. + + ⛔ `core.shares._clean_entries` silently drops junk, and its docstring argues that correctly: + a UI mid-save must not lose the whole list to one malformed row. **But it validates the SHAPE + of a string and the role word — never that the user EXISTS, is ACTIVE, or is in this tenant**, + so a typo'd name is stored, reported as a successful save, and never reaches anybody. The + sharer believes the person has access. That is item 18's plain reading. + ⚠ The correct population is computed THREE FUNCTIONS BELOW and served to the picker + (`_people`). One route, two populations, and the write door was the permissive one. + ⚠ `*` (everyone) is not a user and is admitted deliberately — it is R10's vocabulary for + "every account that can already open the surface". + """ + known = {p["username"].strip().lower() for p in _people(session.tenant)} + unknown = [] + for e in entries or (): + if not isinstance(e, dict): + continue + user = str(e.get("user") or "").strip().lower() + if user and user != shares.EVERYONE and user not in known: + unknown.append(user) + if unknown: + raise err(400, "unknown_people", + "no active account in this workspace is named " + + ", ".join(sorted(set(unknown))) + + ". Nothing was shared. Pick people from the list rather than typing a name.") + + +@router.get("/share/mine") +def my_shares(session: Session = Depends(require_session)): + """Everything shared WITH me, by kind — the "Shared with me" rail section (R10). + + Registered before `/share/{kind}/{oid}` so the literal path wins the match; FastAPI resolves + in declaration order and `mine` would otherwise be read as a `kind`, answering 400 for a URL + that is not malformed at all. + """ + return shares.shared_with(session.uname, st=session.runtime) + + +@router.get("/share/{kind}/{oid}") +def get_share(kind: str, oid: str, session: Session = Depends(require_session)): + kind = _kind_or_400(kind) + rec = shares.grants(kind, oid, st=session.runtime) + role = shares.role_for(kind, oid, session.uname, is_admin=session.admin, st=session.runtime) + may_admin = shares.may_administer(kind, oid, session.uname, is_admin=session.admin, + st=session.runtime) + # W39-T29 — the dialog opens with GET before its first PUT. Until that PUT exists the grant + # registry has no owner to return from `may_administer`, even though the same caller may safely + # claim their own object through the PUT path below. Reflect that exact claim predicate here: + # a Member who owns an unshared View receives the people picker; a collaborator still does not. + if not rec["owner"] and not may_admin and _owns_object(session, kind, oid): + may_admin = True + # ⭐ W32-T26 (audit S-3) — A STRANGER LEARNS NOTHING. This route used to answer for ANY id: + # who owns it, everyone it is granted to, and the tenant's whole username↔name directory — + # to any signed-in session, about objects it cannot open. Now a caller with no role on an + # object must prove they can reach it, and gets a 404 otherwise: the same answer a + # non-existent id gives, so the route cannot be used to probe which ids are real. + # ⚠ `role is None` is the cheap pre-test, so the N-topic resolution below runs only for a + # caller who has no relationship with the object at all. + if role is None and not _can_see_object(session, kind, oid): + raise err(404, "no_object", "no such item, or it is not shared with this account") + return { + **rec, + "role": role, + "mayAdminister": may_admin, + # ⚠ WAVE 21 (C1 identity fix): grant entries BIND on USERNAMES, so the picker must carry + # them. `assignable_people` serves bare display names because `user`-kind CELLS store + # display names — that list's shape cannot change without migrating cell values — so + # this route serves objects of its own. Existing grants that were written as lowercased + # display names are normalised by the wave-21 cleanup script. + # ⭐ W32-T26 (audit S-3) — the roster is the EDITOR's data, so it rides only for a caller + # who may open the editor. A read-only grantee gets the grant list (their fair question is + # "who else has this?") and not a directory of every account in the workspace. + "people": _people(session.tenant) if may_admin else [], + } + + +def _people(tenant): + """[{username, name}] for this tenant — same population as `assignable_people`, with the + BINDING identity alongside the display one.""" + try: + reg = users.registry() or {} + except Exception: + return [] + want = str(tenant or '').strip().lower() + out = [] + for uname, u in reg.items(): + if not isinstance(u, dict) or u.get('active') is False: + continue + if want and str(u.get('tenant') or 'royal-imports').strip().lower() != want: + continue + out.append({"username": str(uname), "name": str(u.get('name') or uname)}) + return sorted(out, key=lambda p: p["name"].lower()) + + +@router.put("/share/{kind}/{oid}") +def put_share(kind: str, oid: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + kind = _kind_or_400(kind) + body = body or {} + rec = shares.grants(kind, oid, st=session.runtime) + # An object with NO grant record yet has no owner — the first person to share it claims it. + # That is safe because reaching this route at all means passing the surface's own wall, and + # the alternative (refusing until somebody seeds an owner) would make a brand-new folder + # unshareable by the person who just made it. + if rec["owner"]: + if not shares.may_administer(kind, oid, session.uname, is_admin=session.admin, + st=session.runtime): + raise err(403, "not_owner", + "only the owner of this item (or an administrator) can change who it is " + "shared with") + # ⛔⛔ W32-T26 (audit S-1) — THE CLAIM NOW HAS A PRECONDITION. An object with no grant record + # is still claimed by the first person to share it — that rule is right, and refusing until + # somebody seeds an owner would make a brand-new folder unshareable by the person who just + # made it. What was missing is the half the old comment ASSERTED and the code never did: the + # claimant has to be able to reach the object. Without this, any signed-in account could + # stamp itself owner of an id it had never seen and lock the real creator out for good. + elif not _owns_object(session, kind, oid): + raise err(404, "no_object", "no such item, or it is not shared with this account") + entries = body.get("entries") + if not isinstance(entries, list): + raise err(400, "bad_entries", + "entries must be a list of {user, role}. Send [] to un-share, which is how " + "revoking is expressed") + _entries_or_400(session, entries) + out = shares.set_grants(kind, oid, entries, owner=rec["owner"] or session.uname, + st=session.runtime) + _notify_new_grantees(session, kind, oid, before=rec["entries"], after=out.get("entries") or []) + return out + + +def _notify_new_grantees(session, kind, oid, before, after): + """⭐⭐ W32-T28 (owner item 18's last clause, contract C3) — tell the RECEIVER, in their Inbox. + + Owner item 18 ends *"being shared a database notifies the receiver"*. Until now sharing was + silent: the grant landed in a rail section the receiver had to notice on their own, which is + why "I shared it with you" and "I never saw it" were both true. + + ⛔ WRITTEN ON THE SHARE, NEVER POLLED. `/notifications` re-evaluates view-ALERTS on read + because an alert is a live question about rows; a share is an EVENT that happened once, and + polling for it would mean re-deriving "was this new?" on every inbox open — the diff below + only exists here, at the moment the set changes. + + ⚠ ONLY THE NEWLY ADDED. `PUT` REPLACES the whole entry set (revoking is expressed by absence), + so every save re-sends everyone who was already there. Diffing against `before` is what stops + a rename or a role change from ringing the bell for people whose access did not change. + ⚠ `*` IS NOT NOTIFIED: there is no user to name, and minting one notification per account in + the tenant on a single click is a broadcast nobody asked for. The rail still shows it. + ⚠ IT NEVER RAISES. A notification that fails must not fail the share that triggered it — the + grant is the user's actual intent, and `core.alerts.notify` writes with `flush='async'`. + """ + try: + was = {e.get("user") for e in (before or ()) if isinstance(e, dict)} + fresh = [str(e.get("user")) for e in (after or ()) + if isinstance(e, dict) and e.get("user") not in was + and e.get("user") != shares.EVERYONE] + if not fresh: + return + import core.alerts as alerts + + label, route, view_id = _object_ref(session, kind, oid) + if not route: + # ⛔ NO ROUTE, NO NOTIFICATION — the receiver would get a row that opens nothing, and + # `notification_view` would have to invent a target. Silence is the honest answer + # here; the rail still shows the grant under "Shared with me". + return + sharer = str(session.user.get("name") or session.uname) + for user in fresh: + # ⚠ THE SHAPE IS `routes_alerts.notification_view`'s SHARE BRANCH, and the two must + # agree or the Inbox row is unclickable: `topic` selects the branch and `key` becomes + # `alertId`, which that branch reads as the id to open. Both constants are IMPORTED + # from there rather than typed again — one vocabulary, one owner. + # ⭐⭐ W33-T28 (`ASK C-14`, answered) — `actor` IS THE SENDER, AND IT IS THE ONLY WAY + # THE INBOX CAN NAME ONE. An alert and an automation have no person behind them and + # are honestly named by their machine; a SHARE has a real person, and only this call + # site knows who. ⛔ It is passed as its OWN field rather than recovered from the + # `detail` prose below: a sender parsed out of " shared this with you" breaks + # the first time the sentence is reworded, silently, in the header + # [[grep-output-is-not-source]]. The prose stays as the body; this is the From. + alerts.notify(user, label, topic=_SHARE_TOPIC, key=route, row_id=view_id, + detail=f"{sharer} shared this with you", actor=sharer, + st=session.runtime) + except Exception: # noqa: BLE001 + return + + +def _object_ref(session, kind, oid): + """`(label, route, view_id)` — what to CALL the shared thing, and where it OPENS. + + ⛔ THE ROUTE IS RESOLVED HERE, NOT SHAPED IN THE CONSUMER, AND THE FIRST VERSION GOT IT + WRONG: it put the raw `oid` in the notification's key, so a shared VIEW produced + `target: {module: "database", id: "view_42"}` — an instruction to open a database named + `view_42`. It read perfectly in the payload and would have opened nothing. **A view is not + addressable on its own; it is a SELECTION inside a topic's grid**, so the pair is what has to + travel. Caught by looking at the notification the driver actually produced, not by reading + the code back. + + ⚠ `label` never falls back to a raw id. A notification headed `ut_leads_3f2a` tells the + receiver nothing they can act on, and the id is already in the target. + ⚠ An unresolvable object answers `route=None`, and the caller then sends NOTHING rather than + a row that opens nowhere. + """ + try: + if kind == "field": + # ⭐⭐ W38-T16 — A COLUMN IS NOT ADDRESSABLE ON ITS OWN, exactly as a view is not: it + # is a column INSIDE a database, so the target that travels is the DATABASE. Without + # this branch the function falls through to the view/folder loop, finds nothing, + # answers `route=None` — and `_notify_new_grantees` returns EARLY. The grant lands and + # the receiver is never told, which is the silent half of owner item 18 reopened one + # kind over. + from routes_alerts import route_for_topic + table_key, field_key = shares.split_field_oid(oid) + if not table_key: + return ("A column", None, "") + try: + import core.shared_overlay as shared_overlay + defn = (shared_overlay.fields(table_key, st=session.runtime) or {}).get(field_key) + except Exception: # noqa: BLE001 + defn = None + label = str((defn or {}).get("label") or "").strip() or field_key + # ⚠ TWO SPELLINGS REACH THIS LINE AND ONE MAP ANSWERS BOTH. `shared_overlay` is keyed + # by whatever the calling door already held: a `ut_*` database uses its bare key, + # while a registry topic uses `_table_workspace` (`product_data.TABLE_KEY`). + # `route_for_topic` speaks the GRID SCOPE vocabulary (`customer`, not + # `customer_data`), so the suffix comes off before it is asked — rather than a second + # route table being written here, which is how the two come apart. + _WS = "_table_workspace" + scope = table_key[:-len(_WS)] if table_key.endswith(_WS) else table_key + return (label, route_for_topic(scope) or None, "") + if kind == "database": + import core.user_tables as ut + defn = (ut.all_defs(st=session.runtime) or {}).get(str(oid)) or {} + # A user table IS its own route key in both vocabularies (`route_for_topic`). + return (str(defn.get("label") or "").strip() or "A database", str(oid), "") + import core.table_store as table_store + from routes_alerts import route_for_topic + for topic in _topics(session): + ops = table_store.make(f"{topic}_table_workspace", st=session.runtime) + hit = ops.find_view(oid) if kind == "view" else ops.find_folder(oid) + if not hit: + continue + route = route_for_topic(topic) + if not route: + break + row = hit[1] if len(hit) > 1 else {} + name = str((row or {}).get("name") or "").strip() + # ⚠ Only a VIEW carries a selection. A folder is a rail grouping, so the target opens + # the grid and stops there rather than naming a view the receiver did not get. + return (name or ("A view" if kind == "view" else "A folder"), + route, str(oid) if kind == "view" else "") + except Exception: # noqa: BLE001 + pass + return ({"view": "A view", "folder": "A folder", + "field": "A column"}.get(kind, "An item"), None, "") diff --git a/api/routes_slack.py b/api/routes_slack.py index 3108f41e6956dfa8f29a3fb8e5db55655b641d5c..4cf017909fc13d069974d6d9cad3d927e8fd9fa5 100644 --- a/api/routes_slack.py +++ b/api/routes_slack.py @@ -1,772 +1,772 @@ -"""routes_slack.py — MANAGE AGENT: a bot per Slack channel, walled by the engine that walls a -person. (wave 33, owner item 10 · `W33-T38` / `W33-T39` / `W33-T67`.) - -Owner, verbatim (2026-08-14): *"Expand Slack we need to be able to configure an AI bot PER channel -whose permissions we can toggle based on Fields etc of the databases, exactly like how we toggle -permissioning per user."* - -──────────────────────────────────────────────────────────────────────────────────────────────── -⛔⛔ D-84 SAID THIS WAS "adapted from OpenTag" AND THAT PREMISE IS FALSE (PRD amendment A1). - -`PENDING.md` D-84 has named OpenTag as the model for per-Slack-channel permissioning since -2026-08-08. **OpenTag has no permission model to adapt.** `app/runtime-host.ts` is -`identifyUser: () => OPENTAG_SERVICE_USER` — ONE constant identity for every request — and its own -`sender-context.ts` says the Slack sender is *"informational only… not gating access."* Its -authorization ceiling is whole-MCP-server, decided once at process boot from env vars. It is a -live specimen of identity that LOOKS like a permission and is not. - -`core/perm_scope.py` is already strictly more expressive: per database, per row, per FIELD, per -principal. So item 10 is OUR engine projected onto a CHANNEL principal — see `agent_principal` -below, which is the whole projection and is nine lines. Study: -`.claude/wiki/research/opentag-adoption.md` (licence gate: MIT, PASSED). - -What OpenTag genuinely contributes is the WRITE-APPROVAL interceptor (`W33-T67`, `approval_card` -below) and nothing else. Its TRANSPORT is AVOID wholesale — hosted CopilotKit Intelligence, a -second Node process, a persistent outbound socket and each tenant's Slack credentials held by a -third party, against ONE process on the HF free tier. The door here is `routes_forms.py`'s proven -unauthenticated-public shape plus Slack's signing-secret check. - -──────────────────────────────────────────────────────────────────────────────────────────────── -⚠ A CHANNEL IS A PRINCIPAL, NOT A PERSON. The wall binds to the Slack conversation id, so every -human in that channel reads through the same rules. That is the honest reading of "a bot per -channel" and the pane says it out loud, because the alternative is an admin assuming per-person -scoping from a screen that looks exactly like the per-person one. - -⚠ AND THE IDENTITY IS THE CHANNEL ID, NEVER ITS NAME. A channel can be renamed; its id cannot. -A wall bound to `#sales` silently re-points the day somebody renames the channel. -""" -import hmac -import json -import os -import secrets -import time -from datetime import datetime, timezone - -from fastapi import APIRouter, Body, Depends, Request - -from deps import Session, err, require_session -from routes_admin import admin_gate - -router = APIRouter(prefix="/api/v1") - -#: The tenant's channel agents: `{id: record}`. A per-tenant bucket, so it rides -#: `runtime.store_key`'s prefix and never lands in tenant #0's namespace. -AGENTS_KEY = "slack_agents" - -#: Pending mutations awaiting a human's click (`W33-T67`). Server-only, per tenant. -PENDING_KEY = "slack_pending_writes" - -#: How long an approval card stays clickable. An approval is a statement about the world as it was -#: when the card was rendered; an hour later the values it names may no longer be the values it -#: would write. Expiring is the honest behaviour, and an expired card says so rather than 403ing. -APPROVAL_TTL_S = 15 * 60 - -#: Slack rejects a replayed request older than 5 minutes; so do we, before any signature work. -SLACK_MAX_SKEW_S = 60 * 5 - -MAX_BODY_BYTES = 64 * 1024 - -#: ⛔ ROW-CAPPED BECAUSE AN APPROVER WHO CANNOT READ THE CARD CANNOT APPROVE IT. Copied -#: deliberately from OpenTag's `agent/write_confirmation.py`, which caps for the same reason: -#: past a certain length Slack collapses a message behind "Show more", and a human clicking -#: Approve on a card whose tail they never saw is worse than no card at all. -APPROVAL_MAX_ROWS = 12 - - -# ── the tenant's agent records ─────────────────────────────────────────────────────────────── -def _agents(rt): - """`{id: record}` for one tenant. `{}` on any failure — an unreadable bucket must degrade to - "this tenant has no agents", never to a 500 on the settings pane.""" - try: - found = rt.get(AGENTS_KEY) or {} - except Exception: # noqa: BLE001 - return {} - return found if isinstance(found, dict) else {} - - -def _clean_agent(raw, prior=None): - """One stored record, built KEY BY KEY. - - ⛔ Never `dict(raw)` and never `**raw`. This record is the SUBJECT of a permission decision; - a client-supplied key landing in it is a client-supplied input to `perm_scope`. The same rule - `routes_forms._public_form` follows on the way out, applied here on the way in. - """ - prior = prior if isinstance(prior, dict) else {} - out = { - "id": str(prior.get("id") or raw.get("id") or ""), - "channel": str(raw.get("channel") or prior.get("channel") or "").strip()[:64], - "channelName": str(raw.get("channelName") or prior.get("channelName") or "").strip()[:80], - "label": " ".join(str(raw.get("label") or prior.get("label") or "").split())[:80], - "active": bool(raw.get("active", prior.get("active", True))), - # ⛔ DEFAULT TRUE, AND `is not False` RATHER THAN TRUTHINESS. A record written before this - # key existed must read as "approval required": the one direction a default here is - # allowed to be wrong in is the safe one, because an unattended write cannot be undone by - # revoking the bot afterwards [[default-must-pass-its-own-guard]]. - "writeApproval": raw.get("writeApproval", prior.get("writeApproval", True)) is not False, - # The permission block. Same shape, same validator, same engine as a user's. - "perms": prior.get("perms") if isinstance(prior.get("perms"), dict) else {}, - "perms_v": int(prior.get("perms_v") or 0), - "createdBy": str(prior.get("createdBy") or raw.get("createdBy") or ""), - "createdAt": str(prior.get("createdAt") or raw.get("createdAt") or ""), - } - return out - - -def _put_agent(session, agent_id, mutate): - """Read-modify-write ONE agent under the tenant's bucket, synchronously. - - `flush="sync"` because the client re-reads the list immediately after every write here, and an - async flush would let the refetch win the race and paint the pre-write state — the - [[refetch-eats-its-own-write]] shape. - """ - def _set(cur): - cur = dict(cur or {}) - rec = cur.get(agent_id) if isinstance(cur.get(agent_id), dict) else None - nxt = mutate(rec) - if nxt is None: - cur.pop(agent_id, None) - else: - cur[agent_id] = nxt - return cur - - session.runtime.update(AGENTS_KEY, _set, flush="sync") - - -# ── ⭐⭐ THE PROJECTION — THE WHOLE OF "the same engine that permissions a user" ─────────────── -def agent_principal(agent): - """A channel agent, as a PRINCIPAL `core/perm_scope` already understands. - - ⭐⭐ THIS IS THE ENTIRE POINT OF THE TICKET AND IT IS NINE LINES. `perm_scope` takes a user - RECORD — a plain dict with `perms`, `perms_v` and `role` — not a User object and not a session. - So a channel agent carrying a `perms` block simply IS such a record, and `may_access`, - `visible_fields`, `hidden_keys` and `apply_row_scope` work on it unmodified. There is no second - engine, no parallel vocabulary, and nothing about a channel to keep in step with anything about - a person. That is what the owner's *"exactly like how we toggle permissioning per user"* means - when it is true in code rather than in appearance. - - ⛔ `role` IS HARDCODED NON-ADMIN AND MUST STAY SO. `perm_scope.may_access` returns True - unconditionally for `role == 'admin'` — the break-glass clause that keeps the owner out of a - locked store. A bot has no such emergency and no hands: an agent record that could carry - `role: 'admin'` would be one stored key away from bypassing every wall on this page. The key is - not copied from the record; it is written here, every time. - - ⛔ AND `perms_v` IS FORCED TO THE CURRENT VERSION, which makes an UNDECLARED database DENY - (C-PERM amendment 4). Without it a fresh agent would fall through to `perms.may_open`'s legacy - grant wall and read `modules: 'all'`-shaped defaults — i.e. a brand-new bot would start with - access to everything. Fail-closed from the first byte. - """ - import core.perm_scope as perm_scope - return { - "username": f"slack:{(agent or {}).get('channel') or '?'}", - "role": "agent", - "perms": (agent or {}).get("perms") or {}, - "perms_v": perm_scope.PERMS_VERSION, - # No `bus`, no `agent`: a channel is not a business unit and not a salesperson. Absent - # rather than 'all' — `perms.allowed_bu_labels` narrows on absence, which is the direction - # this must fail in. - } - - -def agent_may_read(agent, module): - """May this channel open `module` at all?""" - import core.perm_scope as perm_scope - if not (agent or {}).get("active", True): - return False - return bool(perm_scope.may_access(agent_principal(agent), module)) - - -def agent_visible_fields(agent, fields, module): - """`fields` minus this channel's hidden closure — the SAME transitive closure a person gets, - so a formula over a hidden column cannot leak it back through arithmetic.""" - import core.perm_scope as perm_scope - return perm_scope.visible_fields(fields, agent_principal(agent), module) - - -def agent_rows(agent, rows, module, fields, ctx=None): - """The rows this channel may receive — `permits()`, so an unanswerable permanent filter DENIES - rather than being ignored.""" - import core.perm_scope as perm_scope - p = agent_principal(agent) - hide = perm_scope.hidden_keys(p, module, fields) - kept = perm_scope.apply_row_scope(rows, p, module, fields, ctx) - # Both wires, never one: the field list and the row payload are separate, and stripping only - # the first leaves the value sitting in the second where anyone can read it. - return [perm_scope.strip_row(r, hide) for r in kept] - - -# ── the credential ─────────────────────────────────────────────────────────────────────────── -def slack_creds(rt): - """This TENANT's Slack credentials from its own keychain, or None. - - ⛔ NO ENVIRONMENT FALLBACK, DELIBERATELY, AND THIS IS D-202's LESSON APPLIED BEFORE IT - HAPPENS AGAIN. Meta Ads shipped loading end-to-end and was still not a connector, because - `keychain.meta_creds` had no caller and the token came from the owner's `.env` — making it a - tenant-#0 FACT indistinguishable from a screenshot. `keychain.meta_creds`' own docstring spells - out the rule: handing the environment's credential to a tenant whose admin has not stored one - is the leak the resolver exists to prevent. So a tenant with no entry gets None and the pane - SAYS so. - - ⚠ Reads through `list_entries` + `read_fields` (both public) rather than `keychain._first_creds` - (private, and in a file outside this lane's fence). Same deterministic rule: entries are - id-sorted, so "first" is stable across reads rather than dict-order luck. - """ - try: - import core.keychain as keychain - for row in keychain.list_entries(rt): - if str(row.get("type") or "") != "slack": - continue - fields = keychain.read_fields(rt, row.get("id")) - if isinstance(fields, dict) and fields.get("signing_secret"): - return fields - except Exception: # noqa: BLE001 - # A locked or unreadable keychain reads as "not configured" for the PANE, which is honest; - # the signature check below fails closed regardless, so this cannot widen anything. - return None - return None - - -# ── the authenticated doors (the Manage agent surface) ─────────────────────────────────────── -def _summary(agent, modules): - """One sentence per agent for the list row — computed here, because the alternative is one - round trip per row to fill one cell (the same reason `AdminUser.access` exists).""" - # ⭐ W36-T22 / C2 — every listed database is governed now; the `enforced` flag it used to - # filter on is DELETED, and `m.get("enforced")` would have gone silently falsy here and - # summarised every agent as "" (no databases at all). - governed = list(modules) - perms = agent.get("perms") or {} - open_n = sum(1 for m in governed if (perms.get(m["key"]) or {}).get("access")) - if not governed: - return "" - if open_n == 0: - return "No access" - restricted = sum(1 for m in governed - if (perms.get(m["key"]) or {}).get("access") - and ((perms.get(m["key"]) or {}).get("filter") - or (perms.get(m["key"]) or {}).get("hiddenFields"))) - head = (f"All {open_n} database{'' if open_n == 1 else 's'}" if open_n == len(governed) - else f"{open_n} of {len(governed)} databases") - return f"{head}, {restricted} restricted" if restricted else head - - -@router.get("/agents") -def list_channel_agents(session: Session = Depends(admin_gate)): - """This tenant's channel agents, plus whether Slack is reachable at all.""" - import routes_admin - modules = routes_admin._perm_modules(session) - rows = [] - for aid, a in sorted(_agents(session.runtime).items()): - if not isinstance(a, dict): - continue - rows.append({"id": aid, "channel": a.get("channel") or "", - "channelName": a.get("channelName") or "", - "label": a.get("label") or "", "active": a.get("active", True) is not False, - "writeApproval": a.get("writeApproval", True) is not False, - "summary": _summary(a, modules)}) - return {"agents": rows, "configured": slack_creds(session.runtime) is not None} - - -@router.post("/agents") -def create_channel_agent(body: dict = Body(default=None), - session: Session = Depends(admin_gate)): - body = body if isinstance(body, dict) else {} - channel = str(body.get("channel") or "").strip() - if not channel: - raise err(400, "no_channel", "a Slack channel ID is required") - existing = _agents(session.runtime) - # ⚠ ONE AGENT PER CHANNEL. Two records for one conversation would mean two walls, and nothing - # anywhere decides which one applies — the [[one-question-two-normalizers]] shape, in the one - # place where the two answers are "may read" and "may not". - for a in existing.values(): - if isinstance(a, dict) and str(a.get("channel") or "") == channel: - raise err(409, "channel_taken", - "that channel already has an agent — edit it instead of adding a second") - aid = secrets.token_urlsafe(9) - rec = _clean_agent(dict(body, id=aid, createdBy=session.uname, - createdAt=datetime.now(timezone.utc).isoformat(timespec="seconds"))) - _put_agent(session, aid, lambda _prior: rec) - fresh = _agents(session.runtime).get(aid) - if not isinstance(fresh, dict): - # The store took the write and did not record it. A 200 here would tell an administrator - # the agent exists when it does not. - raise err(503, "store_unavailable", "the agent was NOT created") - return {"agent": {"id": aid, "channel": rec["channel"], "channelName": rec["channelName"], - "label": rec["label"], "active": rec["active"], - "writeApproval": rec["writeApproval"], "summary": "No access"}} - - -@router.get("/agents/{agent_id}") -def get_channel_agent(agent_id: str, session: Session = Depends(admin_gate)): - """The agent PLUS the permission envelope — the SAME shape `get_perms` answers with. - - ⭐ ONE PAYLOAD SHAPE FOR TWO ROOMS. `permsModel.parsePermsPayload` parses this, and - `settings/ModulePermsList` renders it, because they are literally the same client code. A - second envelope here would mean a second parser and a second set of defaults within a wave. - """ - import routes_admin - a = _agents(session.runtime).get(str(agent_id)) - if not isinstance(a, dict): - raise err(404, "no_such_agent", "no agent with that id") - modules = routes_admin._perm_modules(session) - # ⭐⭐ W36-T22 / C2 — EVERY listed database, `ut_*` included. ⛔ THIS IS WHY THE FLAG'S - # DELETION IS NOT A ONE-FILE CHANGE: `m.get("enforced")` on a row that no longer carries the - # key is None, so this list would have been EMPTY and the channel perms editor would have - # governed ZERO databases while looking entirely correct. A Slack channel is a principal of - # the SAME wall (`verify_perm_scope` section H) and gets the same catalogue. - governed = [m["key"] for m in modules] - stored = a.get("perms") or {} - principal = agent_principal(a) - import core.perm_scope as perm_scope - perms_out = {} - for k in governed: - e = stored.get(k) - # ⛔ W36-T22 — `may_read`, the same evaluator the read door uses, for the reason spelled - # out at `routes_admin.get_perms`. ⚠ It answers DIFFERENTLY here and correctly so: a - # channel agent is not a person with a share, so a `ut_*` key it has not been granted - # defaults CLOSED — which is the fail-closed direction for a bot, and the same answer the - # table routes would give it. - perms_out[k] = e if isinstance(e, dict) else { - "access": bool(perm_scope.may_read(principal, k, st=session.runtime)), - "filter": None, "hiddenFields": []} - return {"id": str(agent_id), "channel": a.get("channel") or "", - "channelName": a.get("channelName") or "", "label": a.get("label") or "", - "active": a.get("active", True) is not False, - "writeApproval": a.get("writeApproval", True) is not False, - "perms": perms_out, - # ⛔ ALWAYS THE CURRENT VERSION, because `agent_principal` forces it: the editor must - # render "undeclared = denied", not the legacy-grant reading it would show at 0. - "perms_v": perm_scope.PERMS_VERSION, - # A bot is NEVER an admin — see `agent_principal`. Sent so the editor never paints - # the "Everything (admin)" state for a principal that cannot have it. - "is_admin": False, - "modules": modules, - "fields_by_module": {k: routes_admin._module_fields(k, session=session) - for k in governed}} - - -@router.put("/agents/{agent_id}/perms") -def put_channel_agent_perms(agent_id: str, body: dict = Body(default=None), - session: Session = Depends(admin_gate)): - """Replace this agent's permission block wholesale — the same validator a user's block gets.""" - import routes_admin - body = body if isinstance(body, dict) else {} - if not isinstance(_agents(session.runtime).get(str(agent_id)), dict): - raise err(404, "no_such_agent", "no agent with that id") - if "perms" not in body: - raise err(400, "empty_patch", "no perms to save") - mods = routes_admin._perm_modules(session) - # ⭐ THE SAME `_clean_perms`, NOT A COPY OF IT. It refuses a filter this module cannot - # evaluate, refuses a hiddenFields key that names nothing, and refuses a BU condition the - # pushdown cannot read. Every one of those is exactly as true for a channel as for a person, - # and a second validator here is a second place for one of them to go missing. - # ⚠ CORRECTED W36-T22: this list used to end "refuses an unenforced `ut_*` key". That refusal - # is DELETED — W36-T21 armed the wall over every database, so there is no unenforced key left - # to refuse, and a comment naming a rule that no longer exists is how the next wave re-derives - # it ([[two-gates-can-assert-opposite-things]]). - cleaned = routes_admin._clean_perms( - body.get("perms"), - governed_keys={m["key"] for m in mods}, session=session) or {} - - import core.perm_scope as perm_scope - - def _mut(prior): - if not isinstance(prior, dict): - return None - nxt = dict(prior) - nxt["perms"] = cleaned - nxt["perms_v"] = perm_scope.PERMS_VERSION - return nxt - - _put_agent(session, str(agent_id), _mut) - fresh = _agents(session.runtime).get(str(agent_id)) or {} - if (fresh.get("perms") or {}) != cleaned: - # The one direction this must never fail in: telling an administrator the wall is up when - # it is not. - raise err(503, "store_unavailable", "the permissions were not saved — the agent is UNCHANGED") - return {"id": str(agent_id), "perms": fresh.get("perms") or {}, - "perms_v": int(fresh.get("perms_v") or 0)} - - -@router.patch("/agents/{agent_id}") -def patch_channel_agent(agent_id: str, body: dict = Body(default=None), - session: Session = Depends(admin_gate)): - body = body if isinstance(body, dict) else {} - if not isinstance(_agents(session.runtime).get(str(agent_id)), dict): - raise err(404, "no_such_agent", "no agent with that id") - #: ⛔ `perms` IS NOT PATCHABLE HERE. It has its own door with its own validator; accepting it - #: on the metadata route would be a second, unvalidated way to write the wall. - allowed = {k: v for k, v in body.items() - if k in ("label", "active", "writeApproval", "channelName")} - if not allowed: - raise err(400, "empty_patch", "nothing to change") - - def _mut(prior): - return _clean_agent(allowed, prior=prior) if isinstance(prior, dict) else None - - _put_agent(session, str(agent_id), _mut) - import routes_admin - a = _agents(session.runtime).get(str(agent_id)) or {} - return {"agent": {"id": str(agent_id), "channel": a.get("channel") or "", - "channelName": a.get("channelName") or "", "label": a.get("label") or "", - "active": a.get("active", True) is not False, - "writeApproval": a.get("writeApproval", True) is not False, - "summary": _summary(a, routes_admin._perm_modules(session))}} - - -@router.delete("/agents/{agent_id}") -def delete_channel_agent(agent_id: str, session: Session = Depends(admin_gate)): - if not isinstance(_agents(session.runtime).get(str(agent_id)), dict): - raise err(404, "no_such_agent", "no agent with that id") - _put_agent(session, str(agent_id), lambda _prior: None) - if isinstance(_agents(session.runtime).get(str(agent_id)), dict): - raise err(503, "store_unavailable", "the agent was NOT removed") - return {"ok": True} - - -# ── ⭐ W33-T67: THE WRITE-APPROVAL GATE ─────────────────────────────────────────────────────── -def _is_empty(value): - """Is this value ABSENT, as opposed to falsy? - - ⛔ COPIED EXACTLY FROM OpenTag's `agent/write_confirmation.py`, and the exactness is the point: - `0` and `False` are VALUES a human must see on the card. Setting a price to 0 or a flag to - False is precisely the kind of write somebody needs to approve, and a naive `if not value` - drops both rows silently — the approver then reads a card that does not describe the write - they are approving. The MIT licence gate for this borrowing is recorded in - `.claude/wiki/research/opentag-adoption.md`. - """ - return value is None or (isinstance(value, str) and value.strip() == "") - - -def approval_card(action, values, agent=None): - """The card a human reads before a bot writes. `{action, rows, truncated, note}`. - - ⛔ ROW-CAPPED, and the cap is a SAFETY property rather than a layout one: past a certain - length Slack collapses a message behind "Show more", and **an approver who cannot read the - card cannot approve it.** When rows are dropped the card SAYS how many — a truncation nobody - was told about is the violation, not the truncation (wave-30 R6's second sentence). - """ - rows = [{"field": str(k), "value": v} - for k, v in (values or {}).items() if not _is_empty(v)] - rows.sort(key=lambda r: r["field"]) - shown, dropped = rows[:APPROVAL_MAX_ROWS], max(0, len(rows) - APPROVAL_MAX_ROWS) - return { - "action": str(action or "")[:80], - "rows": shown, - "truncated": dropped, - "note": (f"{dropped} more field(s) not shown — open the record to review them before " - f"approving." if dropped else ""), - "agent": (agent or {}).get("label") or (agent or {}).get("channel") or "", - } - - -def intercept_write(agent, action, values, stash): - """THE INTERCEPTOR. Returns `("commit", None)` or `("await_approval", card)`. - - ⭐ The one thing worth taking from OpenTag: a MUTATING tool call is halted, rendered as a card - naming the action and its values, and committed only on an explicit human accept. Everything - else in that repo — the transport, the runtime, the identity model — is AVOID (amendment A1). - - ⛔ FAIL-CLOSED ON A MISSING FLAG: `is not False`, so a record written before `writeApproval` - existed requires approval. The unsafe direction here is not recoverable — revoking a bot does - not un-write what it wrote. - """ - if (agent or {}).get("writeApproval", True) is not False: - card = approval_card(action, values, agent) - stash(card) - return "await_approval", card - return "commit", None - - -def _pending(rt): - try: - found = rt.get(PENDING_KEY) or {} - except Exception: # noqa: BLE001 - return {} - return found if isinstance(found, dict) else {} - - -def stash_pending(rt, agent, action, values, now=None): - """Store one awaiting-approval mutation and return its token. - - The token is what rides the card's button, so it is `secrets.token_urlsafe` and not the - action's id: a guessable value on an unauthenticated door is a way to approve somebody else's - write. Server-only bucket, per tenant, exactly as `routes_forms.TOKENS_KEY` is. - """ - token = secrets.token_urlsafe(18) - row = {"at": float(now if now is not None else time.time()), - "channel": str((agent or {}).get("channel") or ""), - "action": str(action or "")[:80], - "values": values if isinstance(values, dict) else {}} - rt.update(PENDING_KEY, lambda cur: {**(cur or {}), token: row}, flush="sync") - return token - - -#: `{action kind: fn(rt, row) -> None}`. Registered by whatever can actually perform a mutation. -#: -#: ⛔⛔ EMPTY IN PRODUCTION TODAY, AND SAYING SO IS THE POINT. `commit_pending` below is the ONE -#: place an approved write may be performed, so it is the place a negative control can bite — but -#: a gate is only as honest as what it guards. Until something registers here, an approved card -#: performs NOTHING and `commit_pending` returns `no_committer` rather than pretending. That is a -#: declared absence, not a silent one [[flag-shipped-without-its-writer]]. -_COMMITTERS = {} - - -def register_committer(kind, fn): - """Declare who may perform an approved mutation of `kind`. Idempotent by key.""" - _COMMITTERS[str(kind)] = fn - - -def commit_pending(rt, token, approved, now=None): - """Resolve ONE approval card. Returns one of - `expired` · `unknown` · `rejected` · `no_committer` · `committed` · `failed`. - - ⛔⛔ THIS IS THE WRITE GATE, AND IT IS A SEPARATE FUNCTION FROM THE ROUTE ON PURPOSE. T67's - `done-when` asks for an NC proving a REJECTED card writes nothing. A control aimed at the - route could not bite while the route had no write in it at all — it would stay green with the - guard deleted, because there would be nothing for the guard to be protecting anything from - ([[gate-can-report-green-on-nothing]], and a verifier caught exactly that on this ticket's - first draft). Putting the commit behind ONE named seam gives the control a real subject: the - gate registers a recorder, and `rejected` must leave it untouched while `approve` must call it - exactly once. - - ⛔ THE TOKEN IS CONSUMED BEFORE EITHER BRANCH, and before any committer runs. A card that - survives its own click is a replayable write, and "approve twice" must never mean "write - twice". The delete is `flush="sync"` for the same reason. - """ - now = float(now if now is not None else time.time()) - row = _pending(rt).get(str(token)) - if not isinstance(row, dict): - return "unknown" - # Consume FIRST — before the TTL verdict, before the approve/reject branch, before any write. - rt.update(PENDING_KEY, - lambda cur: {k: v for k, v in (cur or {}).items() if k != str(token)}, - flush="sync") - if now - float(row.get("at") or 0) > APPROVAL_TTL_S: - return "expired" - if not approved: - # ⛔ NOTHING BELOW THIS LINE RUNS FOR A REJECTED CARD. The whole gate is this return. - return "rejected" - fn = _COMMITTERS.get(str(row.get("action") or "")) - if fn is None: - return "no_committer" - try: - fn(rt, row) - except Exception: # noqa: BLE001 - # A committer that raises must not read as a commit. The card is already consumed, so the - # honest report is that it failed, and the human asks again. - return "failed" - return "committed" - - -# ── the UNAUTHENTICATED Slack door ─────────────────────────────────────────────────────────── -#: Sliding window, keyed on the SLACK TEAM, never the client IP. -#: -#: ⛔ WHY NOT THE IP, WHICH IS WHAT `routes_forms` DOES. Every Slack event arrives from Slack's own -#: infrastructure, so an IP window is one shared bucket for every workspace on the platform: one -#: busy tenant would spend the allowance for all of them, and the symptom would be another -#: tenant's bot going quiet. The team id is the closest thing to the actual noisy party. -#: ⚠ It is attacker-CONTROLLED before the signature is checked, so the window is applied AFTER -#: verification, never before — an unsigned request is refused on cost grounds anyway (a signature -#: check is a single HMAC). -RATE_WINDOW_S, RATE_PER_WINDOW = 60, 60 -_HITS: dict = {} - - -def _rate_ok(key, now): - seen = [t for t in _HITS.get(key, ()) if now - t < RATE_WINDOW_S] - seen.append(now) - _HITS[key] = seen - if len(_HITS) > 4096: - for k in [k for k, v in _HITS.items() if not v or now - v[-1] > RATE_WINDOW_S]: - _HITS.pop(k, None) - return len(seen) <= RATE_PER_WINDOW - - -async def _raw_body(request): - """The body as RAW BYTES, capped. - - ⛔⛔ RAW, NOT PARSED, AND THAT IS NOT A STYLE CHOICE. Slack signs the literal string - `v0:{timestamp}:{body}` byte for byte. `routes_forms._bounded_body` — the function this is - otherwise copied from — returns only the parsed dict and DISCARDS the bytes, so a verbatim - copy of it could not verify a Slack signature at all: re-serialising the dict changes - whitespace and key order and the HMAC no longer matches. The bug would look like "Slack - signatures are always invalid", which is indistinguishable from a wrong secret. - - ⚠ And it streams rather than calling `request.body()`: FastAPI would otherwise have buffered - the whole body before the handler's first line, so a content-length check inside the handler - caps nothing (`routes_forms`' own scar, recorded in its docstring). - """ - size, chunks = 0, [] - async for chunk in request.stream(): - size += len(chunk) - if size > MAX_BODY_BYTES: - raise err(413, "body_too_large", "that request is too large") - chunks.append(chunk) - return b"".join(chunks) - - -def _refuse(): - """ONE refusal for every reason. No oracle. - - A wrong signature, a stale timestamp, an unknown team, an unconfigured tenant and a channel - with no agent all answer identically — otherwise the door tells an unauthenticated caller - which tenants exist and which channels are configured, which is the question it was built to - not answer. `routes_forms._refuse` takes the same posture for the same reason. - """ - return err(403, "bad_request", "that request could not be verified") - - -def _verify(secret, timestamp, raw, signature, now): - """Slack's v0 signature, with the replay window checked FIRST.""" - try: - ts = int(str(timestamp or "0")) - except ValueError: - return False - if abs(now - ts) > SLACK_MAX_SKEW_S: - return False - base = b"v0:" + str(ts).encode("ascii") + b":" + (raw or b"") - import hashlib - expected = "v0=" + hmac.new(str(secret).encode("utf-8"), base, hashlib.sha256).hexdigest() - return hmac.compare_digest(expected, str(signature or "")) - - -def _resolve_tenant(raw, timestamp, signature, now): - """`(slug, runtime, creds)` for the tenant whose signing secret verifies this request. - - ⛔ THE TENANT IS DECIDED BY THE SIGNATURE, NEVER BY THE PAYLOAD. A body-supplied `team_id` - would let an unauthenticated caller name the tenant it wants to be — the exact widening - `deps._user_for`'s tenant-equality check closes on the authenticated side. So every configured - tenant's secret is tried and the one that VERIFIES names the tenant. - - ⚠ NO EARLY `break` ON A FAILED CANDIDATE and no per-tenant error: the loop's cost must not - depend on which tenant matched. `routes_forms._resolve` walks tenants the same way, for the - same reason — a public door has no session to ask. - """ - from harness import runtime as _rt - hit = None - try: - slugs = list(_rt.known_tenants()) - except Exception: # noqa: BLE001 - return None, None, None - for slug in slugs: - try: - rt = _rt.get_runtime(slug) - except Exception: # noqa: BLE001 - continue - creds = slack_creds(rt) - if not creds: - continue - if _verify(creds.get("signing_secret"), timestamp, raw, signature, now) and hit is None: - hit = (slug, rt, creds) - return hit if hit else (None, None, None) - - -@router.post("/slack/events") -async def slack_events(request: Request): - """Slack's Events API. UNAUTHENTICATED BY CONSTRUCTION — no `Depends(require_session)`. - - ⭐ Built like `routes_forms.py`'s public door and not like OpenTag's: one process, no vendor, - no second runtime, no persistent outbound socket, and this tenant's Slack secret never leaves - this deployment (amendment A1). - """ - now = time.time() - raw = await _raw_body(request) - ts = request.headers.get("x-slack-request-timestamp") - sig = request.headers.get("x-slack-signature") - slug, rt, _creds = _resolve_tenant(raw, ts, sig, now) - if not rt: - raise _refuse() - - try: - payload = json.loads(raw or b"{}") - except ValueError: - raise _refuse() - if not isinstance(payload, dict): - raise _refuse() - - # Slack's one-time endpoint handshake. Answered ONLY after the signature verified — an - # unsigned challenge echo would confirm the endpoint exists to anyone who probes it. - if payload.get("type") == "url_verification": - return {"challenge": str(payload.get("challenge") or "")[:512]} - - team = str(payload.get("team_id") or slug) - if not _rate_ok(f"{slug}:{team}", now): - raise err(429, "too_many_requests", "too many requests — wait a moment and try again") - - event = payload.get("event") if isinstance(payload.get("event"), dict) else {} - channel = str(event.get("channel") or "") - agent = None - for a in _agents(rt).values(): - if isinstance(a, dict) and str(a.get("channel") or "") == channel: - agent = a - break - # ⛔ NO AGENT, OR AN INACTIVE ONE, IS SILENCE — a 200 with no action. Not a 403: Slack retries - # a non-2xx up to three times, so refusing here would turn "this channel is not configured" - # into three refusals per message, and the retry storm would be the only visible symptom. - # ⚠ And it must not say WHICH — a distinguishable answer tells an unauthenticated caller which - # channels this workspace has configured. - if not isinstance(agent, dict) or agent.get("active", True) is False: - return {"ok": True} - - # ⛔⛔ THE ANSWERING HALF IS NOT BUILT YET, AND SAYING SO IS THE POINT. What ships here is the - # DOOR and the WALL: a verified request, resolved to a tenant by signature, matched to a - # channel agent whose `perms` block is a `perm_scope` principal (`agent_principal`). Composing - # a reply means an LLM call on the cheap-first ladder plus the read path, and a half-built - # answerer that returns something plausible is worse than one that returns nothing. - # Booked rather than faked [[flag-shipped-without-its-writer]]. - return {"ok": True} - - -@router.post("/slack/interact") -async def slack_interact(request: Request): - """The approval card's button (`W33-T67`). UNAUTHENTICATED, signature-verified, same shape.""" - now = time.time() - raw = await _raw_body(request) - slug, rt, _creds = _resolve_tenant(raw, request.headers.get("x-slack-request-timestamp"), - request.headers.get("x-slack-signature"), now) - if not rt: - raise _refuse() - # Slack posts interactions as `application/x-www-form-urlencoded` with a `payload=` field. - from urllib.parse import parse_qs - try: - form = parse_qs(raw.decode("utf-8")) - payload = json.loads((form.get("payload") or ["{}"])[0]) - except Exception: # noqa: BLE001 - raise _refuse() - if not isinstance(payload, dict): - raise _refuse() - if not _rate_ok(f"{slug}:interact", now): - raise err(429, "too_many_requests", "too many requests — wait a moment and try again") - - actions = payload.get("actions") if isinstance(payload.get("actions"), list) else [] - choice = (actions[0] if actions and isinstance(actions[0], dict) else {}) - token = str(choice.get("value") or "") - # ⭐ ANYTHING THAT IS NOT LITERALLY "approve" IS A REJECT. Fail-closed on a malformed, - # truncated or unknown action id: the direction that can be wrong here without anyone - # noticing is the one that writes. - approved = str(choice.get("action_id") or "") == "approve" - # ⛔ THE ROUTE DOES NOT DECIDE — `commit_pending` does, and it is the ONE seam a negative - # control can aim at. Consumption, the TTL, the reject branch and the committer lookup all - # live there; this handler only turns its verdict into a sentence. - verdict = commit_pending(rt, token, approved, now=now) - return {"text": { - # An unknown, already-used or expired token is not an error: a person clicking a stale - # card should be told it is stale, not shown a failure. - "unknown": "That approval is no longer available. Ask the bot again.", - "expired": "That approval expired before it was answered. Ask the bot again.", - "rejected": "Rejected. Nothing was changed.", - # ⚠ HONEST, not reassuring: the answering half that would register a committer is booked, - # so an approved card today performs nothing and says exactly that rather than "Approved." - "no_committer": "Approved — but this workspace has nothing configured to carry it out yet.", - "failed": "That change could not be completed. Nothing was saved; ask the bot again.", - "committed": "Approved.", - }.get(verdict, "That approval is no longer available. Ask the bot again.")} - - -@router.get("/slack/health") -def slack_health(session: Session = Depends(require_session)): - """Is Slack configured for THIS tenant? Session-gated, and it answers about one tenant only — - enumerating the others would answer a question about our customer list.""" - creds = slack_creds(session.runtime) - return {"configured": creds is not None, - "agents": len(_agents(session.runtime)), - "hasBotToken": bool((creds or {}).get("bot_token"))} +"""routes_slack.py — MANAGE AGENT: a bot per Slack channel, walled by the engine that walls a +person. (wave 33, owner item 10 · `W33-T38` / `W33-T39` / `W33-T67`.) + +Owner, verbatim (2026-08-14): *"Expand Slack we need to be able to configure an AI bot PER channel +whose permissions we can toggle based on Fields etc of the databases, exactly like how we toggle +permissioning per user."* + +──────────────────────────────────────────────────────────────────────────────────────────────── +⛔⛔ D-84 SAID THIS WAS "adapted from OpenTag" AND THAT PREMISE IS FALSE (PRD amendment A1). + +`PENDING.md` D-84 has named OpenTag as the model for per-Slack-channel permissioning since +2026-08-08. **OpenTag has no permission model to adapt.** `app/runtime-host.ts` is +`identifyUser: () => OPENTAG_SERVICE_USER` — ONE constant identity for every request — and its own +`sender-context.ts` says the Slack sender is *"informational only… not gating access."* Its +authorization ceiling is whole-MCP-server, decided once at process boot from env vars. It is a +live specimen of identity that LOOKS like a permission and is not. + +`core/perm_scope.py` is already strictly more expressive: per database, per row, per FIELD, per +principal. So item 10 is OUR engine projected onto a CHANNEL principal — see `agent_principal` +below, which is the whole projection and is nine lines. Study: +`.claude/wiki/research/opentag-adoption.md` (licence gate: MIT, PASSED). + +What OpenTag genuinely contributes is the WRITE-APPROVAL interceptor (`W33-T67`, `approval_card` +below) and nothing else. Its TRANSPORT is AVOID wholesale — hosted CopilotKit Intelligence, a +second Node process, a persistent outbound socket and each tenant's Slack credentials held by a +third party, against ONE process on the HF free tier. The door here is `routes_forms.py`'s proven +unauthenticated-public shape plus Slack's signing-secret check. + +──────────────────────────────────────────────────────────────────────────────────────────────── +⚠ A CHANNEL IS A PRINCIPAL, NOT A PERSON. The wall binds to the Slack conversation id, so every +human in that channel reads through the same rules. That is the honest reading of "a bot per +channel" and the pane says it out loud, because the alternative is an admin assuming per-person +scoping from a screen that looks exactly like the per-person one. + +⚠ AND THE IDENTITY IS THE CHANNEL ID, NEVER ITS NAME. A channel can be renamed; its id cannot. +A wall bound to `#sales` silently re-points the day somebody renames the channel. +""" +import hmac +import json +import os +import secrets +import time +from datetime import datetime, timezone + +from fastapi import APIRouter, Body, Depends, Request + +from deps import Session, err, require_session +from routes_admin import admin_gate + +router = APIRouter(prefix="/api/v1") + +#: The tenant's channel agents: `{id: record}`. A per-tenant bucket, so it rides +#: `runtime.store_key`'s prefix and never lands in tenant #0's namespace. +AGENTS_KEY = "slack_agents" + +#: Pending mutations awaiting a human's click (`W33-T67`). Server-only, per tenant. +PENDING_KEY = "slack_pending_writes" + +#: How long an approval card stays clickable. An approval is a statement about the world as it was +#: when the card was rendered; an hour later the values it names may no longer be the values it +#: would write. Expiring is the honest behaviour, and an expired card says so rather than 403ing. +APPROVAL_TTL_S = 15 * 60 + +#: Slack rejects a replayed request older than 5 minutes; so do we, before any signature work. +SLACK_MAX_SKEW_S = 60 * 5 + +MAX_BODY_BYTES = 64 * 1024 + +#: ⛔ ROW-CAPPED BECAUSE AN APPROVER WHO CANNOT READ THE CARD CANNOT APPROVE IT. Copied +#: deliberately from OpenTag's `agent/write_confirmation.py`, which caps for the same reason: +#: past a certain length Slack collapses a message behind "Show more", and a human clicking +#: Approve on a card whose tail they never saw is worse than no card at all. +APPROVAL_MAX_ROWS = 12 + + +# ── the tenant's agent records ─────────────────────────────────────────────────────────────── +def _agents(rt): + """`{id: record}` for one tenant. `{}` on any failure — an unreadable bucket must degrade to + "this tenant has no agents", never to a 500 on the settings pane.""" + try: + found = rt.get(AGENTS_KEY) or {} + except Exception: # noqa: BLE001 + return {} + return found if isinstance(found, dict) else {} + + +def _clean_agent(raw, prior=None): + """One stored record, built KEY BY KEY. + + ⛔ Never `dict(raw)` and never `**raw`. This record is the SUBJECT of a permission decision; + a client-supplied key landing in it is a client-supplied input to `perm_scope`. The same rule + `routes_forms._public_form` follows on the way out, applied here on the way in. + """ + prior = prior if isinstance(prior, dict) else {} + out = { + "id": str(prior.get("id") or raw.get("id") or ""), + "channel": str(raw.get("channel") or prior.get("channel") or "").strip()[:64], + "channelName": str(raw.get("channelName") or prior.get("channelName") or "").strip()[:80], + "label": " ".join(str(raw.get("label") or prior.get("label") or "").split())[:80], + "active": bool(raw.get("active", prior.get("active", True))), + # ⛔ DEFAULT TRUE, AND `is not False` RATHER THAN TRUTHINESS. A record written before this + # key existed must read as "approval required": the one direction a default here is + # allowed to be wrong in is the safe one, because an unattended write cannot be undone by + # revoking the bot afterwards [[default-must-pass-its-own-guard]]. + "writeApproval": raw.get("writeApproval", prior.get("writeApproval", True)) is not False, + # The permission block. Same shape, same validator, same engine as a user's. + "perms": prior.get("perms") if isinstance(prior.get("perms"), dict) else {}, + "perms_v": int(prior.get("perms_v") or 0), + "createdBy": str(prior.get("createdBy") or raw.get("createdBy") or ""), + "createdAt": str(prior.get("createdAt") or raw.get("createdAt") or ""), + } + return out + + +def _put_agent(session, agent_id, mutate): + """Read-modify-write ONE agent under the tenant's bucket, synchronously. + + `flush="sync"` because the client re-reads the list immediately after every write here, and an + async flush would let the refetch win the race and paint the pre-write state — the + [[refetch-eats-its-own-write]] shape. + """ + def _set(cur): + cur = dict(cur or {}) + rec = cur.get(agent_id) if isinstance(cur.get(agent_id), dict) else None + nxt = mutate(rec) + if nxt is None: + cur.pop(agent_id, None) + else: + cur[agent_id] = nxt + return cur + + session.runtime.update(AGENTS_KEY, _set, flush="sync") + + +# ── ⭐⭐ THE PROJECTION — THE WHOLE OF "the same engine that permissions a user" ─────────────── +def agent_principal(agent): + """A channel agent, as a PRINCIPAL `core/perm_scope` already understands. + + ⭐⭐ THIS IS THE ENTIRE POINT OF THE TICKET AND IT IS NINE LINES. `perm_scope` takes a user + RECORD — a plain dict with `perms`, `perms_v` and `role` — not a User object and not a session. + So a channel agent carrying a `perms` block simply IS such a record, and `may_access`, + `visible_fields`, `hidden_keys` and `apply_row_scope` work on it unmodified. There is no second + engine, no parallel vocabulary, and nothing about a channel to keep in step with anything about + a person. That is what the owner's *"exactly like how we toggle permissioning per user"* means + when it is true in code rather than in appearance. + + ⛔ `role` IS HARDCODED NON-ADMIN AND MUST STAY SO. `perm_scope.may_access` returns True + unconditionally for `role == 'admin'` — the break-glass clause that keeps the owner out of a + locked store. A bot has no such emergency and no hands: an agent record that could carry + `role: 'admin'` would be one stored key away from bypassing every wall on this page. The key is + not copied from the record; it is written here, every time. + + ⛔ AND `perms_v` IS FORCED TO THE CURRENT VERSION, which makes an UNDECLARED database DENY + (C-PERM amendment 4). Without it a fresh agent would fall through to `perms.may_open`'s legacy + grant wall and read `modules: 'all'`-shaped defaults — i.e. a brand-new bot would start with + access to everything. Fail-closed from the first byte. + """ + import core.perm_scope as perm_scope + return { + "username": f"slack:{(agent or {}).get('channel') or '?'}", + "role": "agent", + "perms": (agent or {}).get("perms") or {}, + "perms_v": perm_scope.PERMS_VERSION, + # No `bus`, no `agent`: a channel is not a business unit and not a salesperson. Absent + # rather than 'all' — `perms.allowed_bu_labels` narrows on absence, which is the direction + # this must fail in. + } + + +def agent_may_read(agent, module): + """May this channel open `module` at all?""" + import core.perm_scope as perm_scope + if not (agent or {}).get("active", True): + return False + return bool(perm_scope.may_access(agent_principal(agent), module)) + + +def agent_visible_fields(agent, fields, module): + """`fields` minus this channel's hidden closure — the SAME transitive closure a person gets, + so a formula over a hidden column cannot leak it back through arithmetic.""" + import core.perm_scope as perm_scope + return perm_scope.visible_fields(fields, agent_principal(agent), module) + + +def agent_rows(agent, rows, module, fields, ctx=None): + """The rows this channel may receive — `permits()`, so an unanswerable permanent filter DENIES + rather than being ignored.""" + import core.perm_scope as perm_scope + p = agent_principal(agent) + hide = perm_scope.hidden_keys(p, module, fields) + kept = perm_scope.apply_row_scope(rows, p, module, fields, ctx) + # Both wires, never one: the field list and the row payload are separate, and stripping only + # the first leaves the value sitting in the second where anyone can read it. + return [perm_scope.strip_row(r, hide) for r in kept] + + +# ── the credential ─────────────────────────────────────────────────────────────────────────── +def slack_creds(rt): + """This TENANT's Slack credentials from its own keychain, or None. + + ⛔ NO ENVIRONMENT FALLBACK, DELIBERATELY, AND THIS IS D-202's LESSON APPLIED BEFORE IT + HAPPENS AGAIN. Meta Ads shipped loading end-to-end and was still not a connector, because + `keychain.meta_creds` had no caller and the token came from the owner's `.env` — making it a + tenant-#0 FACT indistinguishable from a screenshot. `keychain.meta_creds`' own docstring spells + out the rule: handing the environment's credential to a tenant whose admin has not stored one + is the leak the resolver exists to prevent. So a tenant with no entry gets None and the pane + SAYS so. + + ⚠ Reads through `list_entries` + `read_fields` (both public) rather than `keychain._first_creds` + (private, and in a file outside this lane's fence). Same deterministic rule: entries are + id-sorted, so "first" is stable across reads rather than dict-order luck. + """ + try: + import core.keychain as keychain + for row in keychain.list_entries(rt): + if str(row.get("type") or "") != "slack": + continue + fields = keychain.read_fields(rt, row.get("id")) + if isinstance(fields, dict) and fields.get("signing_secret"): + return fields + except Exception: # noqa: BLE001 + # A locked or unreadable keychain reads as "not configured" for the PANE, which is honest; + # the signature check below fails closed regardless, so this cannot widen anything. + return None + return None + + +# ── the authenticated doors (the Manage agent surface) ─────────────────────────────────────── +def _summary(agent, modules): + """One sentence per agent for the list row — computed here, because the alternative is one + round trip per row to fill one cell (the same reason `AdminUser.access` exists).""" + # ⭐ W36-T22 / C2 — every listed database is governed now; the `enforced` flag it used to + # filter on is DELETED, and `m.get("enforced")` would have gone silently falsy here and + # summarised every agent as "" (no databases at all). + governed = list(modules) + perms = agent.get("perms") or {} + open_n = sum(1 for m in governed if (perms.get(m["key"]) or {}).get("access")) + if not governed: + return "" + if open_n == 0: + return "No access" + restricted = sum(1 for m in governed + if (perms.get(m["key"]) or {}).get("access") + and ((perms.get(m["key"]) or {}).get("filter") + or (perms.get(m["key"]) or {}).get("hiddenFields"))) + head = (f"All {open_n} database{'' if open_n == 1 else 's'}" if open_n == len(governed) + else f"{open_n} of {len(governed)} databases") + return f"{head}, {restricted} restricted" if restricted else head + + +@router.get("/agents") +def list_channel_agents(session: Session = Depends(admin_gate)): + """This tenant's channel agents, plus whether Slack is reachable at all.""" + import routes_admin + modules = routes_admin._perm_modules(session) + rows = [] + for aid, a in sorted(_agents(session.runtime).items()): + if not isinstance(a, dict): + continue + rows.append({"id": aid, "channel": a.get("channel") or "", + "channelName": a.get("channelName") or "", + "label": a.get("label") or "", "active": a.get("active", True) is not False, + "writeApproval": a.get("writeApproval", True) is not False, + "summary": _summary(a, modules)}) + return {"agents": rows, "configured": slack_creds(session.runtime) is not None} + + +@router.post("/agents") +def create_channel_agent(body: dict = Body(default=None), + session: Session = Depends(admin_gate)): + body = body if isinstance(body, dict) else {} + channel = str(body.get("channel") or "").strip() + if not channel: + raise err(400, "no_channel", "a Slack channel ID is required") + existing = _agents(session.runtime) + # ⚠ ONE AGENT PER CHANNEL. Two records for one conversation would mean two walls, and nothing + # anywhere decides which one applies — the [[one-question-two-normalizers]] shape, in the one + # place where the two answers are "may read" and "may not". + for a in existing.values(): + if isinstance(a, dict) and str(a.get("channel") or "") == channel: + raise err(409, "channel_taken", + "that channel already has an agent — edit it instead of adding a second") + aid = secrets.token_urlsafe(9) + rec = _clean_agent(dict(body, id=aid, createdBy=session.uname, + createdAt=datetime.now(timezone.utc).isoformat(timespec="seconds"))) + _put_agent(session, aid, lambda _prior: rec) + fresh = _agents(session.runtime).get(aid) + if not isinstance(fresh, dict): + # The store took the write and did not record it. A 200 here would tell an administrator + # the agent exists when it does not. + raise err(503, "store_unavailable", "the agent was NOT created") + return {"agent": {"id": aid, "channel": rec["channel"], "channelName": rec["channelName"], + "label": rec["label"], "active": rec["active"], + "writeApproval": rec["writeApproval"], "summary": "No access"}} + + +@router.get("/agents/{agent_id}") +def get_channel_agent(agent_id: str, session: Session = Depends(admin_gate)): + """The agent PLUS the permission envelope — the SAME shape `get_perms` answers with. + + ⭐ ONE PAYLOAD SHAPE FOR TWO ROOMS. `permsModel.parsePermsPayload` parses this, and + `settings/ModulePermsList` renders it, because they are literally the same client code. A + second envelope here would mean a second parser and a second set of defaults within a wave. + """ + import routes_admin + a = _agents(session.runtime).get(str(agent_id)) + if not isinstance(a, dict): + raise err(404, "no_such_agent", "no agent with that id") + modules = routes_admin._perm_modules(session) + # ⭐⭐ W36-T22 / C2 — EVERY listed database, `ut_*` included. ⛔ THIS IS WHY THE FLAG'S + # DELETION IS NOT A ONE-FILE CHANGE: `m.get("enforced")` on a row that no longer carries the + # key is None, so this list would have been EMPTY and the channel perms editor would have + # governed ZERO databases while looking entirely correct. A Slack channel is a principal of + # the SAME wall (`verify_perm_scope` section H) and gets the same catalogue. + governed = [m["key"] for m in modules] + stored = a.get("perms") or {} + principal = agent_principal(a) + import core.perm_scope as perm_scope + perms_out = {} + for k in governed: + e = stored.get(k) + # ⛔ W36-T22 — `may_read`, the same evaluator the read door uses, for the reason spelled + # out at `routes_admin.get_perms`. ⚠ It answers DIFFERENTLY here and correctly so: a + # channel agent is not a person with a share, so a `ut_*` key it has not been granted + # defaults CLOSED — which is the fail-closed direction for a bot, and the same answer the + # table routes would give it. + perms_out[k] = e if isinstance(e, dict) else { + "access": bool(perm_scope.may_read(principal, k, st=session.runtime)), + "filter": None, "hiddenFields": []} + return {"id": str(agent_id), "channel": a.get("channel") or "", + "channelName": a.get("channelName") or "", "label": a.get("label") or "", + "active": a.get("active", True) is not False, + "writeApproval": a.get("writeApproval", True) is not False, + "perms": perms_out, + # ⛔ ALWAYS THE CURRENT VERSION, because `agent_principal` forces it: the editor must + # render "undeclared = denied", not the legacy-grant reading it would show at 0. + "perms_v": perm_scope.PERMS_VERSION, + # A bot is NEVER an admin — see `agent_principal`. Sent so the editor never paints + # the "Everything (admin)" state for a principal that cannot have it. + "is_admin": False, + "modules": modules, + "fields_by_module": {k: routes_admin._module_fields(k, session=session) + for k in governed}} + + +@router.put("/agents/{agent_id}/perms") +def put_channel_agent_perms(agent_id: str, body: dict = Body(default=None), + session: Session = Depends(admin_gate)): + """Replace this agent's permission block wholesale — the same validator a user's block gets.""" + import routes_admin + body = body if isinstance(body, dict) else {} + if not isinstance(_agents(session.runtime).get(str(agent_id)), dict): + raise err(404, "no_such_agent", "no agent with that id") + if "perms" not in body: + raise err(400, "empty_patch", "no perms to save") + mods = routes_admin._perm_modules(session) + # ⭐ THE SAME `_clean_perms`, NOT A COPY OF IT. It refuses a filter this module cannot + # evaluate, refuses a hiddenFields key that names nothing, and refuses a BU condition the + # pushdown cannot read. Every one of those is exactly as true for a channel as for a person, + # and a second validator here is a second place for one of them to go missing. + # ⚠ CORRECTED W36-T22: this list used to end "refuses an unenforced `ut_*` key". That refusal + # is DELETED — W36-T21 armed the wall over every database, so there is no unenforced key left + # to refuse, and a comment naming a rule that no longer exists is how the next wave re-derives + # it ([[two-gates-can-assert-opposite-things]]). + cleaned = routes_admin._clean_perms( + body.get("perms"), + governed_keys={m["key"] for m in mods}, session=session) or {} + + import core.perm_scope as perm_scope + + def _mut(prior): + if not isinstance(prior, dict): + return None + nxt = dict(prior) + nxt["perms"] = cleaned + nxt["perms_v"] = perm_scope.PERMS_VERSION + return nxt + + _put_agent(session, str(agent_id), _mut) + fresh = _agents(session.runtime).get(str(agent_id)) or {} + if (fresh.get("perms") or {}) != cleaned: + # The one direction this must never fail in: telling an administrator the wall is up when + # it is not. + raise err(503, "store_unavailable", "the permissions were not saved — the agent is UNCHANGED") + return {"id": str(agent_id), "perms": fresh.get("perms") or {}, + "perms_v": int(fresh.get("perms_v") or 0)} + + +@router.patch("/agents/{agent_id}") +def patch_channel_agent(agent_id: str, body: dict = Body(default=None), + session: Session = Depends(admin_gate)): + body = body if isinstance(body, dict) else {} + if not isinstance(_agents(session.runtime).get(str(agent_id)), dict): + raise err(404, "no_such_agent", "no agent with that id") + #: ⛔ `perms` IS NOT PATCHABLE HERE. It has its own door with its own validator; accepting it + #: on the metadata route would be a second, unvalidated way to write the wall. + allowed = {k: v for k, v in body.items() + if k in ("label", "active", "writeApproval", "channelName")} + if not allowed: + raise err(400, "empty_patch", "nothing to change") + + def _mut(prior): + return _clean_agent(allowed, prior=prior) if isinstance(prior, dict) else None + + _put_agent(session, str(agent_id), _mut) + import routes_admin + a = _agents(session.runtime).get(str(agent_id)) or {} + return {"agent": {"id": str(agent_id), "channel": a.get("channel") or "", + "channelName": a.get("channelName") or "", "label": a.get("label") or "", + "active": a.get("active", True) is not False, + "writeApproval": a.get("writeApproval", True) is not False, + "summary": _summary(a, routes_admin._perm_modules(session))}} + + +@router.delete("/agents/{agent_id}") +def delete_channel_agent(agent_id: str, session: Session = Depends(admin_gate)): + if not isinstance(_agents(session.runtime).get(str(agent_id)), dict): + raise err(404, "no_such_agent", "no agent with that id") + _put_agent(session, str(agent_id), lambda _prior: None) + if isinstance(_agents(session.runtime).get(str(agent_id)), dict): + raise err(503, "store_unavailable", "the agent was NOT removed") + return {"ok": True} + + +# ── ⭐ W33-T67: THE WRITE-APPROVAL GATE ─────────────────────────────────────────────────────── +def _is_empty(value): + """Is this value ABSENT, as opposed to falsy? + + ⛔ COPIED EXACTLY FROM OpenTag's `agent/write_confirmation.py`, and the exactness is the point: + `0` and `False` are VALUES a human must see on the card. Setting a price to 0 or a flag to + False is precisely the kind of write somebody needs to approve, and a naive `if not value` + drops both rows silently — the approver then reads a card that does not describe the write + they are approving. The MIT licence gate for this borrowing is recorded in + `.claude/wiki/research/opentag-adoption.md`. + """ + return value is None or (isinstance(value, str) and value.strip() == "") + + +def approval_card(action, values, agent=None): + """The card a human reads before a bot writes. `{action, rows, truncated, note}`. + + ⛔ ROW-CAPPED, and the cap is a SAFETY property rather than a layout one: past a certain + length Slack collapses a message behind "Show more", and **an approver who cannot read the + card cannot approve it.** When rows are dropped the card SAYS how many — a truncation nobody + was told about is the violation, not the truncation (wave-30 R6's second sentence). + """ + rows = [{"field": str(k), "value": v} + for k, v in (values or {}).items() if not _is_empty(v)] + rows.sort(key=lambda r: r["field"]) + shown, dropped = rows[:APPROVAL_MAX_ROWS], max(0, len(rows) - APPROVAL_MAX_ROWS) + return { + "action": str(action or "")[:80], + "rows": shown, + "truncated": dropped, + "note": (f"{dropped} more field(s) not shown — open the record to review them before " + f"approving." if dropped else ""), + "agent": (agent or {}).get("label") or (agent or {}).get("channel") or "", + } + + +def intercept_write(agent, action, values, stash): + """THE INTERCEPTOR. Returns `("commit", None)` or `("await_approval", card)`. + + ⭐ The one thing worth taking from OpenTag: a MUTATING tool call is halted, rendered as a card + naming the action and its values, and committed only on an explicit human accept. Everything + else in that repo — the transport, the runtime, the identity model — is AVOID (amendment A1). + + ⛔ FAIL-CLOSED ON A MISSING FLAG: `is not False`, so a record written before `writeApproval` + existed requires approval. The unsafe direction here is not recoverable — revoking a bot does + not un-write what it wrote. + """ + if (agent or {}).get("writeApproval", True) is not False: + card = approval_card(action, values, agent) + stash(card) + return "await_approval", card + return "commit", None + + +def _pending(rt): + try: + found = rt.get(PENDING_KEY) or {} + except Exception: # noqa: BLE001 + return {} + return found if isinstance(found, dict) else {} + + +def stash_pending(rt, agent, action, values, now=None): + """Store one awaiting-approval mutation and return its token. + + The token is what rides the card's button, so it is `secrets.token_urlsafe` and not the + action's id: a guessable value on an unauthenticated door is a way to approve somebody else's + write. Server-only bucket, per tenant, exactly as `routes_forms.TOKENS_KEY` is. + """ + token = secrets.token_urlsafe(18) + row = {"at": float(now if now is not None else time.time()), + "channel": str((agent or {}).get("channel") or ""), + "action": str(action or "")[:80], + "values": values if isinstance(values, dict) else {}} + rt.update(PENDING_KEY, lambda cur: {**(cur or {}), token: row}, flush="sync") + return token + + +#: `{action kind: fn(rt, row) -> None}`. Registered by whatever can actually perform a mutation. +#: +#: ⛔⛔ EMPTY IN PRODUCTION TODAY, AND SAYING SO IS THE POINT. `commit_pending` below is the ONE +#: place an approved write may be performed, so it is the place a negative control can bite — but +#: a gate is only as honest as what it guards. Until something registers here, an approved card +#: performs NOTHING and `commit_pending` returns `no_committer` rather than pretending. That is a +#: declared absence, not a silent one [[flag-shipped-without-its-writer]]. +_COMMITTERS = {} + + +def register_committer(kind, fn): + """Declare who may perform an approved mutation of `kind`. Idempotent by key.""" + _COMMITTERS[str(kind)] = fn + + +def commit_pending(rt, token, approved, now=None): + """Resolve ONE approval card. Returns one of + `expired` · `unknown` · `rejected` · `no_committer` · `committed` · `failed`. + + ⛔⛔ THIS IS THE WRITE GATE, AND IT IS A SEPARATE FUNCTION FROM THE ROUTE ON PURPOSE. T67's + `done-when` asks for an NC proving a REJECTED card writes nothing. A control aimed at the + route could not bite while the route had no write in it at all — it would stay green with the + guard deleted, because there would be nothing for the guard to be protecting anything from + ([[gate-can-report-green-on-nothing]], and a verifier caught exactly that on this ticket's + first draft). Putting the commit behind ONE named seam gives the control a real subject: the + gate registers a recorder, and `rejected` must leave it untouched while `approve` must call it + exactly once. + + ⛔ THE TOKEN IS CONSUMED BEFORE EITHER BRANCH, and before any committer runs. A card that + survives its own click is a replayable write, and "approve twice" must never mean "write + twice". The delete is `flush="sync"` for the same reason. + """ + now = float(now if now is not None else time.time()) + row = _pending(rt).get(str(token)) + if not isinstance(row, dict): + return "unknown" + # Consume FIRST — before the TTL verdict, before the approve/reject branch, before any write. + rt.update(PENDING_KEY, + lambda cur: {k: v for k, v in (cur or {}).items() if k != str(token)}, + flush="sync") + if now - float(row.get("at") or 0) > APPROVAL_TTL_S: + return "expired" + if not approved: + # ⛔ NOTHING BELOW THIS LINE RUNS FOR A REJECTED CARD. The whole gate is this return. + return "rejected" + fn = _COMMITTERS.get(str(row.get("action") or "")) + if fn is None: + return "no_committer" + try: + fn(rt, row) + except Exception: # noqa: BLE001 + # A committer that raises must not read as a commit. The card is already consumed, so the + # honest report is that it failed, and the human asks again. + return "failed" + return "committed" + + +# ── the UNAUTHENTICATED Slack door ─────────────────────────────────────────────────────────── +#: Sliding window, keyed on the SLACK TEAM, never the client IP. +#: +#: ⛔ WHY NOT THE IP, WHICH IS WHAT `routes_forms` DOES. Every Slack event arrives from Slack's own +#: infrastructure, so an IP window is one shared bucket for every workspace on the platform: one +#: busy tenant would spend the allowance for all of them, and the symptom would be another +#: tenant's bot going quiet. The team id is the closest thing to the actual noisy party. +#: ⚠ It is attacker-CONTROLLED before the signature is checked, so the window is applied AFTER +#: verification, never before — an unsigned request is refused on cost grounds anyway (a signature +#: check is a single HMAC). +RATE_WINDOW_S, RATE_PER_WINDOW = 60, 60 +_HITS: dict = {} + + +def _rate_ok(key, now): + seen = [t for t in _HITS.get(key, ()) if now - t < RATE_WINDOW_S] + seen.append(now) + _HITS[key] = seen + if len(_HITS) > 4096: + for k in [k for k, v in _HITS.items() if not v or now - v[-1] > RATE_WINDOW_S]: + _HITS.pop(k, None) + return len(seen) <= RATE_PER_WINDOW + + +async def _raw_body(request): + """The body as RAW BYTES, capped. + + ⛔⛔ RAW, NOT PARSED, AND THAT IS NOT A STYLE CHOICE. Slack signs the literal string + `v0:{timestamp}:{body}` byte for byte. `routes_forms._bounded_body` — the function this is + otherwise copied from — returns only the parsed dict and DISCARDS the bytes, so a verbatim + copy of it could not verify a Slack signature at all: re-serialising the dict changes + whitespace and key order and the HMAC no longer matches. The bug would look like "Slack + signatures are always invalid", which is indistinguishable from a wrong secret. + + ⚠ And it streams rather than calling `request.body()`: FastAPI would otherwise have buffered + the whole body before the handler's first line, so a content-length check inside the handler + caps nothing (`routes_forms`' own scar, recorded in its docstring). + """ + size, chunks = 0, [] + async for chunk in request.stream(): + size += len(chunk) + if size > MAX_BODY_BYTES: + raise err(413, "body_too_large", "that request is too large") + chunks.append(chunk) + return b"".join(chunks) + + +def _refuse(): + """ONE refusal for every reason. No oracle. + + A wrong signature, a stale timestamp, an unknown team, an unconfigured tenant and a channel + with no agent all answer identically — otherwise the door tells an unauthenticated caller + which tenants exist and which channels are configured, which is the question it was built to + not answer. `routes_forms._refuse` takes the same posture for the same reason. + """ + return err(403, "bad_request", "that request could not be verified") + + +def _verify(secret, timestamp, raw, signature, now): + """Slack's v0 signature, with the replay window checked FIRST.""" + try: + ts = int(str(timestamp or "0")) + except ValueError: + return False + if abs(now - ts) > SLACK_MAX_SKEW_S: + return False + base = b"v0:" + str(ts).encode("ascii") + b":" + (raw or b"") + import hashlib + expected = "v0=" + hmac.new(str(secret).encode("utf-8"), base, hashlib.sha256).hexdigest() + return hmac.compare_digest(expected, str(signature or "")) + + +def _resolve_tenant(raw, timestamp, signature, now): + """`(slug, runtime, creds)` for the tenant whose signing secret verifies this request. + + ⛔ THE TENANT IS DECIDED BY THE SIGNATURE, NEVER BY THE PAYLOAD. A body-supplied `team_id` + would let an unauthenticated caller name the tenant it wants to be — the exact widening + `deps._user_for`'s tenant-equality check closes on the authenticated side. So every configured + tenant's secret is tried and the one that VERIFIES names the tenant. + + ⚠ NO EARLY `break` ON A FAILED CANDIDATE and no per-tenant error: the loop's cost must not + depend on which tenant matched. `routes_forms._resolve` walks tenants the same way, for the + same reason — a public door has no session to ask. + """ + from harness import runtime as _rt + hit = None + try: + slugs = list(_rt.known_tenants()) + except Exception: # noqa: BLE001 + return None, None, None + for slug in slugs: + try: + rt = _rt.get_runtime(slug) + except Exception: # noqa: BLE001 + continue + creds = slack_creds(rt) + if not creds: + continue + if _verify(creds.get("signing_secret"), timestamp, raw, signature, now) and hit is None: + hit = (slug, rt, creds) + return hit if hit else (None, None, None) + + +@router.post("/slack/events") +async def slack_events(request: Request): + """Slack's Events API. UNAUTHENTICATED BY CONSTRUCTION — no `Depends(require_session)`. + + ⭐ Built like `routes_forms.py`'s public door and not like OpenTag's: one process, no vendor, + no second runtime, no persistent outbound socket, and this tenant's Slack secret never leaves + this deployment (amendment A1). + """ + now = time.time() + raw = await _raw_body(request) + ts = request.headers.get("x-slack-request-timestamp") + sig = request.headers.get("x-slack-signature") + slug, rt, _creds = _resolve_tenant(raw, ts, sig, now) + if not rt: + raise _refuse() + + try: + payload = json.loads(raw or b"{}") + except ValueError: + raise _refuse() + if not isinstance(payload, dict): + raise _refuse() + + # Slack's one-time endpoint handshake. Answered ONLY after the signature verified — an + # unsigned challenge echo would confirm the endpoint exists to anyone who probes it. + if payload.get("type") == "url_verification": + return {"challenge": str(payload.get("challenge") or "")[:512]} + + team = str(payload.get("team_id") or slug) + if not _rate_ok(f"{slug}:{team}", now): + raise err(429, "too_many_requests", "too many requests — wait a moment and try again") + + event = payload.get("event") if isinstance(payload.get("event"), dict) else {} + channel = str(event.get("channel") or "") + agent = None + for a in _agents(rt).values(): + if isinstance(a, dict) and str(a.get("channel") or "") == channel: + agent = a + break + # ⛔ NO AGENT, OR AN INACTIVE ONE, IS SILENCE — a 200 with no action. Not a 403: Slack retries + # a non-2xx up to three times, so refusing here would turn "this channel is not configured" + # into three refusals per message, and the retry storm would be the only visible symptom. + # ⚠ And it must not say WHICH — a distinguishable answer tells an unauthenticated caller which + # channels this workspace has configured. + if not isinstance(agent, dict) or agent.get("active", True) is False: + return {"ok": True} + + # ⛔⛔ THE ANSWERING HALF IS NOT BUILT YET, AND SAYING SO IS THE POINT. What ships here is the + # DOOR and the WALL: a verified request, resolved to a tenant by signature, matched to a + # channel agent whose `perms` block is a `perm_scope` principal (`agent_principal`). Composing + # a reply means an LLM call on the cheap-first ladder plus the read path, and a half-built + # answerer that returns something plausible is worse than one that returns nothing. + # Booked rather than faked [[flag-shipped-without-its-writer]]. + return {"ok": True} + + +@router.post("/slack/interact") +async def slack_interact(request: Request): + """The approval card's button (`W33-T67`). UNAUTHENTICATED, signature-verified, same shape.""" + now = time.time() + raw = await _raw_body(request) + slug, rt, _creds = _resolve_tenant(raw, request.headers.get("x-slack-request-timestamp"), + request.headers.get("x-slack-signature"), now) + if not rt: + raise _refuse() + # Slack posts interactions as `application/x-www-form-urlencoded` with a `payload=` field. + from urllib.parse import parse_qs + try: + form = parse_qs(raw.decode("utf-8")) + payload = json.loads((form.get("payload") or ["{}"])[0]) + except Exception: # noqa: BLE001 + raise _refuse() + if not isinstance(payload, dict): + raise _refuse() + if not _rate_ok(f"{slug}:interact", now): + raise err(429, "too_many_requests", "too many requests — wait a moment and try again") + + actions = payload.get("actions") if isinstance(payload.get("actions"), list) else [] + choice = (actions[0] if actions and isinstance(actions[0], dict) else {}) + token = str(choice.get("value") or "") + # ⭐ ANYTHING THAT IS NOT LITERALLY "approve" IS A REJECT. Fail-closed on a malformed, + # truncated or unknown action id: the direction that can be wrong here without anyone + # noticing is the one that writes. + approved = str(choice.get("action_id") or "") == "approve" + # ⛔ THE ROUTE DOES NOT DECIDE — `commit_pending` does, and it is the ONE seam a negative + # control can aim at. Consumption, the TTL, the reject branch and the committer lookup all + # live there; this handler only turns its verdict into a sentence. + verdict = commit_pending(rt, token, approved, now=now) + return {"text": { + # An unknown, already-used or expired token is not an error: a person clicking a stale + # card should be told it is stale, not shown a failure. + "unknown": "That approval is no longer available. Ask the bot again.", + "expired": "That approval expired before it was answered. Ask the bot again.", + "rejected": "Rejected. Nothing was changed.", + # ⚠ HONEST, not reassuring: the answering half that would register a committer is booked, + # so an approved card today performs nothing and says exactly that rather than "Approved." + "no_committer": "Approved — but this workspace has nothing configured to carry it out yet.", + "failed": "That change could not be completed. Nothing was saved; ask the bot again.", + "committed": "Approved.", + }.get(verdict, "That approval is no longer available. Ask the bot again.")} + + +@router.get("/slack/health") +def slack_health(session: Session = Depends(require_session)): + """Is Slack configured for THIS tenant? Session-gated, and it answers about one tenant only — + enumerating the others would answer a question about our customer list.""" + creds = slack_creds(session.runtime) + return {"configured": creds is not None, + "agents": len(_agents(session.runtime)), + "hasBotToken": bool((creds or {}).get("bot_token"))} diff --git a/api/routes_statements.py b/api/routes_statements.py index c39730724d527120c094a5afc94652703bb1de87..c2cbedf8619c07e441bc5a6f1f80ffb5982b0772 100644 --- a/api/routes_statements.py +++ b/api/routes_statements.py @@ -1,298 +1,298 @@ -"""routes_statements.py — the statement-of-account sender, ported off Streamlit (EXIT-6). - -⛔ THIS IS THE ONE SANCTIONED ODOO WRITER IN THE ENTIRE SYSTEM. Everything else in AIOS is -read-only on Odoo by hard block. `modules/collections_send.py` owns a narrow client that whitelists -exactly `mail.mail create` (queueing an outbound email) and nothing else; these routes call it and -add no write of their own. - -WHY IT EXISTS SEPARATELY FROM THE COLLECTIONS PAGE. Owner ruling wave-17 item 15 retired the -Collections *dashboard* — the worklist is the shared "Collections" view on the Customer grid, from -the same reconciled blocks. What that ruling explicitly left "untouched" is this send workflow, so -when `app.py` is deleted it is the ONLY live Streamlit-only feature, and it moves here rather than -dying with the host. Ported faithfully: same tiers, same filters, same template placeholders, same -preview, same test-send, same two-step confirm. - -THE THREE GUARDRAILS, and where each is enforced: - 1. SAFE_MODE (default ON) — in the DATA LAYER (`collections_send.queue_statement`), so no route, - payload or UI can bypass it. These routes only REPORT it; they never re-implement the check. - 2. Admin only — `admin_gate` (role, fail-closed), mirroring the Streamlit `if not is_admin()`. - 3. ⭐ TENANT — NEW HERE, and it did not exist in Streamlit because it could not. `cs_mod.Odoo()` - reads Odoo credentials from the ENVIRONMENT, which after the keychain cutover belongs to - TENANT #0 ALONE. On the multi-tenant API an unguarded route would let a nurilab or gtmlab - admin queue mail from Royal Imports' Odoo, as Royal Imports. `_royal_only` closes that; the - single-tenant Streamlit host never had the exposure, so this is a port that must ADD a wall - rather than copy one. - -NOTHING SENDS ON A GET. The send route requires an explicit customer list in the body; there is no -"send all" parameter, deliberately — the confirm step is a product requirement, not a formality. -""" -from fastapi import APIRouter, Body, Depends - -from deps import Session, err -from routes_admin import admin_gate - -# W35-T35 (C8): the tenant predicate, held once. See `_royal_only` for why the import runs this -# way round. `main.py` already imports both modules, so this adds no load. -import automation_engine as engine - -router = APIRouter(prefix="/api/v1") - -#: Cache the Odoo follow-up pull briefly. The Streamlit page used `@st.cache_data(ttl=1800)`; the -#: list moves slowly (it is a dunning worklist, not a live feed) and the pull is a multi-model read. -_TTL = 1800 -_cache = {"at": 0.0, "rows": None} - - -def _cs(): - import modules.collections_send as cs - return cs - - -def _royal_only(session: Session) -> Session: - """⛔ See the module docstring, guardrail 3. The send client is env-credentialed, so it is - tenant #0's and only tenant #0's. Refuse for anyone else rather than send as the wrong company. - - Keyed on the runtime, never on a request field: a tenant is a property of the SESSION. - - ⭐⭐ WAVE 35 · T35 / CONTRACT C8 — THE TEST ITSELF NOW LIVES IN ONE PLACE. Statements became an - agent step this wave, so the automation engine has to answer the same question at two more - doors (which tenants see the action, which tenants may store it). C8's words are "the predicate - is imported, never re-expressed", and this is the direction that import can run: the engine - imports no route module and no FastAPI, so reaching `_royal_only` FROM it would drag `deps` and - `routes_admin` into a route-free module. The engine holds the boolean; this holds the refusal. - ⚠ WHAT STAYS HERE IS THE HTTP SHAPE, and that is deliberate — a 404 rather than a 403, so the - surface is invisible rather than forbidden. The engine must not know about status codes.""" - if not engine.is_statement_tenant(session.runtime): - raise err(404, "not_found", "statements are not configured for this workspace") - return session - - -def _gate(session: Session = Depends(admin_gate)) -> Session: - return _royal_only(session) - - -def _rows(force=False): - import time - cs = _cs() - if force or _cache["rows"] is None or (time.time() - _cache["at"]) > _TTL: - _cache["rows"] = cs.load_collection_list(cs.Odoo()) - _cache["at"] = time.time() - return _cache["rows"], time.strftime("%Y-%m-%d %H:%M", time.localtime(_cache["at"])) - - -def _public(row): - """Strip the internals the Streamlit grid also hid (`_`-prefixed + partner_id is kept, because - the client needs a stable row identity that is not the display name).""" - return {k: v for k, v in row.items() if not str(k).startswith("_")} - - -@router.get("/admin/statements") -def statements(refresh: int = 0, session: Session = Depends(_gate)): - """The worklist + everything the sender UI needs to render itself honestly.""" - cs = _cs() - try: - rows, loaded_at = _rows(force=bool(refresh)) - except Exception as e: - raise err(502, "odoo_unavailable", f"could not load the collection list: {str(e)[:200]}") - return { - "rows": [_public(r) for r in rows], - "loadedAt": loaded_at, - # The guardrail is reported, never decided, here — the data layer owns it. - "safeMode": bool(cs.SAFE_MODE), - "safeRecipients": sorted(cs.SAFE_RECIPIENTS), - "sender": {"name": cs.SENDER_NAME, "email": cs.SENDER_EMAIL, - "replyTo": cs.REPLY_TO, "company": cs.COMPANY}, - "templates": {"subject": cs.DEFAULT_SUBJECT, "intro": cs.DEFAULT_INTRO, - "footer": cs.DEFAULT_FOOTER}, - # W35-T35: ONE literal, read from the engine, so the sender's worklist and a statements - # agent's tier filter cannot come to mean different things. - "tiers": list(engine.STATEMENT_TIERS), - } - - -def _find(rows, customer): - return next((r for r in rows if r.get("Customer") == customer), None) - - -@router.post("/admin/statements/preview") -def preview(body: dict = Body(default=None), session: Session = Depends(_gate)): - """Render ONE customer's statement exactly as the send path would.""" - cs = _cs() - body = body or {} - rows, _ = _rows() - row = _find(rows, body.get("customer")) - if row is None: - raise err(404, "not_found", "no such customer on the collection list") - t = body.get("templates") or {} - import datetime as dt - month = dt.date.today().strftime("%B %Y") - subject = (t.get("subject") or cs.DEFAULT_SUBJECT) - try: - subject = subject.format(customer=row["Customer"], company=cs.COMPANY, month=month) - except Exception: # noqa: BLE001 - # An unknown placeholder is the user's typo, not a 500. Show the template verbatim so they - # can see what they typed rather than getting an opaque error. - # ⛔ W36-T42: THIS CAUGHT THREE OF THE WAYS `str.format` FAILS AND THERE ARE MORE. - # `{customer` is a ValueError, `{customer.x}` an AttributeError, `{customer:%Y}` a - # TypeError, and every one of them is the same user mistake this branch was written for. - # Naming a subset of the exception types turns a typo in the OTHER half into a 500 on the - # one screen somebody opens to avoid mailing 200 people the wrong thing. - pass - return { - "html": cs.render_statement_html(row, t.get("intro") or cs.DEFAULT_INTRO, - t.get("footer") or cs.DEFAULT_FOOTER), - "to": row.get("Email") or "", - "subject": subject, - } - - -@router.post("/admin/statements/send") -def send(body: dict = Body(default=None), session: Session = Depends(_gate)): - """Queue statements. Returns per-customer outcomes — NEVER a bare count. - - `overrideTo` is the test-send path: one customer, one address. SAFE_MODE still applies (the - data layer refuses an address outside the allow-list), which is why the route does not check it. - """ - cs = _cs() - body = body or {} - names = [str(n) for n in (body.get("customers") or []) if str(n).strip()] - if not names: - raise err(400, "bad_request", "name at least one customer") - # ⛔⛔ W36-T42 / D-299 — A DECLARED TEST SEND WITH NO ADDRESS REFUSES. IT DOES NOT BECOME A - # REAL ONE. This read `(body.get("overrideTo") or "").strip() or None`, so a caller who sent - # `overrideTo: ""` (the field left blank, the state not yet typed into, a trimmed-away space) - # got a REAL statement mailed to the debtor's own address, reported back as `test: false`. On - # the one route in this product that is allowed to write to Odoo, the difference between a - # rehearsal and mailing a live customer was one empty string. - # - # ⭐ THE DISCRIMINATOR IS THE CALLER'S DECLARATION, NOT THE VALUE. A body with no `overrideTo` - # key and no `test` flag is the PRODUCTION send, and refusing that would delete the feature - # D-299 exists to preserve. What can be refused is a caller who SAID this is a test: the key - # being present, or `test: true`, is that statement, and an empty address beside it is the - # mistake. So both spellings of "absent" are covered: the key present and blank, and `test` - # asserted with no key at all. - # - # ⚠ THIS IS THE SECOND OF THREE WALLS AND THE ONLY ONE A PAYLOAD CANNOT ROUTE AROUND. - # `automationApi.testSendStatement` refuses an empty address before the request is built, and - # SAFE_MODE refuses an address outside the allow-list inside `queue_statement`. The client - # wall is bypassable by construction (anything can POST); this one is not. - declared_test = ("overrideTo" in body) or bool(body.get("test")) - override = str(body.get("overrideTo") or "").strip() - if declared_test and not override: - raise err(400, "no_override_address", - "a test send needs the address to send the test to. Without one this would " - "mail the customer's own address, which is the opposite of a test") - override = override or None - if override and len(names) != 1: - raise err(400, "bad_request", "a test send takes exactly one customer") - t = body.get("templates") or {} - rows, _ = _rows() - - sent, failed, skipped = [], [], [] - for name in names: - row = _find(rows, name) - if row is None: - failed.append({"customer": name, "error": "not on the current collection list"}) - continue - if not override and not row.get("Email"): - # The Streamlit page warned and skipped these. Reporting them SEPARATELY from failures - # keeps "we could not" distinct from "there was nowhere to send". - skipped.append({"customer": name, "reason": "no email address on the customer record"}) - continue - try: - mid = cs.queue_statement(cs.Odoo(), row, t.get("subject") or cs.DEFAULT_SUBJECT, - t.get("intro") or cs.DEFAULT_INTRO, - t.get("footer") or cs.DEFAULT_FOOTER, override_to=override) - sent.append({"customer": name, "to": override or row.get("Email"), "mailId": mid}) - except Exception as e: - # SafeModeBlocked lands here too, and that is correct: to the caller a guardrail refusal - # and an Odoo error are both "this one did not go", each with its own honest message. - failed.append({"customer": name, "error": str(e)[:200]}) - return {"sent": sent, "failed": failed, "skipped": skipped, - "safeMode": bool(cs.SAFE_MODE), "test": bool(override)} - - -# --------------------------------------------------------------------------------------------- -# WAVE 35 · T36 / CONTRACT C8 / OWNER RULING R10 — THE AGENT'S PARKED BATCH. -# -# ⛔⛔ THE AGENT ASSEMBLES; A PERSON CLICKS SEND. `automation_engine.run_statements` renders a batch -# and parks it on the automation definition, and it imports no mail path at all. THIS is the only -# door that releases one, and it is deliberately built out of the parts already here: -# · `_gate` — `admin_gate` THEN `_royal_only`, byte-for-byte the dependency the other three -# endpoints use. Not a copy of the checks: the same object. -# · `cs.queue_statement` — the same call `send()` above makes, so SAFE_MODE's allow-list refusal -# happens in the DATA LAYER, where no route, payload or UI can reach around it. -# ⇒ Nothing about the guardrails is re-expressed here, which is what makes "unchanged" checkable. -# -# ⚠ WHY THE BATCH IS RE-RENDERED FROM THE LIVE WORKLIST RATHER THAN SENT AS STORED. The parked -# `html` is what a person APPROVED and is what the review screen shows; but a balance can move -# between the parking and the click, and mailing a figure we know to be stale is worse than mailing -# a fresh one. So the parked batch decides WHO and WITH WHAT WORDS, and the live row decides the -# NUMBERS — the same split `preview` already makes. A customer who has left the worklist entirely -# (they paid) is reported as skipped rather than invoiced. - -@router.get("/admin/statements/agent/{auto_id}") -def agent_batch(auto_id: str, session: Session = Depends(_gate)): - """What is parked and waiting for a click, for ONE agent. `null` when nothing is.""" - defn = (engine.all_definitions(session.runtime) or {}).get(str(auto_id)) or {} - pending = defn.get("pendingStatements") - if not isinstance(pending, dict): - return {"pending": None} - # ⚠ The rendered `html` is NOT returned in the list — a 200-statement batch of rendered mail is - # megabytes, and the review screen needs the count and the names to decide. `preview` already - # renders ONE on demand. - return {"pending": { - "ts": pending.get("ts"), "count": int(pending.get("count") or 0), - "notes": list(pending.get("notes") or []), - "items": [{k: v for k, v in it.items() if k != "html"} - for it in (pending.get("items") or [])], - "safeMode": bool(_cs().SAFE_MODE)}} - - -@router.post("/admin/statements/agent/{auto_id}/send") -def agent_send(auto_id: str, body: dict = Body(default=None), - session: Session = Depends(_gate)): - """Release a parked batch. THE CLICK R10 REQUIRES — nothing else in the system calls this. - - ⛔ IT REFUSES AN EMPTY OR MISSING BATCH rather than answering 200 with nothing sent: a send - that reports success and mails nobody is the `view_upsert` failure mode (200 OK, zero writes) - arriving on the one route in this system that talks to a mail server. - """ - cs = _cs() - defn = (engine.all_definitions(session.runtime) or {}).get(str(auto_id)) or {} - pending = defn.get("pendingStatements") - items = list((pending or {}).get("items") or []) if isinstance(pending, dict) else [] - if not items: - raise err(404, "no_batch", "there is nothing parked for this agent to send") - # ⚠ An explicit subset is allowed (a person unticking a customer on the review screen) but it - # may only NARROW the parked batch. A name that was never parked cannot be introduced by the - # payload, or the review stops being what authorised the send. - want = {str(n) for n in (body or {}).get("customers") or []} - if want: - items = [it for it in items if str(it.get("customer")) in want] - if not items: - raise err(400, "bad_request", "none of those customers are in the parked batch") - rows, _ = _rows() - sent, failed, skipped = [], [], [] - for it in items: - name = str(it.get("customer") or "") - row = _find(rows, name) - if row is None: - skipped.append({"customer": name, - "reason": "no longer on the collection list, so nothing is owed"}) - continue - try: - # ⛔ THE SAME DATA-LAYER CALL `send()` MAKES. SafeModeBlocked is raised INSIDE - # `queue_statement`, so the guardrail cannot be argued with from here. - mid = cs.queue_statement(cs.Odoo(), row, - str(it.get("subject") or "") or cs.DEFAULT_SUBJECT, - cs.DEFAULT_INTRO, cs.DEFAULT_FOOTER) - sent.append({"customer": name, "to": row.get("Email"), "mailId": mid}) - except Exception as e: # noqa: BLE001 - failed.append({"customer": name, "error": str(e)[:200]}) - # ⚠ CLEARED ONLY WHEN NOTHING IS LEFT TO RETRY. A batch dropped while some of it failed would - # lose the list of who still needs a statement, and nobody would know to look. - if sent and not failed: - engine.clear_statements(session.runtime, auto_id) - return {"sent": sent, "failed": failed, "skipped": skipped, - "safeMode": bool(cs.SAFE_MODE), "cleared": bool(sent and not failed)} +"""routes_statements.py — the statement-of-account sender, ported off Streamlit (EXIT-6). + +⛔ THIS IS THE ONE SANCTIONED ODOO WRITER IN THE ENTIRE SYSTEM. Everything else in AIOS is +read-only on Odoo by hard block. `modules/collections_send.py` owns a narrow client that whitelists +exactly `mail.mail create` (queueing an outbound email) and nothing else; these routes call it and +add no write of their own. + +WHY IT EXISTS SEPARATELY FROM THE COLLECTIONS PAGE. Owner ruling wave-17 item 15 retired the +Collections *dashboard* — the worklist is the shared "Collections" view on the Customer grid, from +the same reconciled blocks. What that ruling explicitly left "untouched" is this send workflow, so +when `app.py` is deleted it is the ONLY live Streamlit-only feature, and it moves here rather than +dying with the host. Ported faithfully: same tiers, same filters, same template placeholders, same +preview, same test-send, same two-step confirm. + +THE THREE GUARDRAILS, and where each is enforced: + 1. SAFE_MODE (default ON) — in the DATA LAYER (`collections_send.queue_statement`), so no route, + payload or UI can bypass it. These routes only REPORT it; they never re-implement the check. + 2. Admin only — `admin_gate` (role, fail-closed), mirroring the Streamlit `if not is_admin()`. + 3. ⭐ TENANT — NEW HERE, and it did not exist in Streamlit because it could not. `cs_mod.Odoo()` + reads Odoo credentials from the ENVIRONMENT, which after the keychain cutover belongs to + TENANT #0 ALONE. On the multi-tenant API an unguarded route would let a nurilab or gtmlab + admin queue mail from Royal Imports' Odoo, as Royal Imports. `_royal_only` closes that; the + single-tenant Streamlit host never had the exposure, so this is a port that must ADD a wall + rather than copy one. + +NOTHING SENDS ON A GET. The send route requires an explicit customer list in the body; there is no +"send all" parameter, deliberately — the confirm step is a product requirement, not a formality. +""" +from fastapi import APIRouter, Body, Depends + +from deps import Session, err +from routes_admin import admin_gate + +# W35-T35 (C8): the tenant predicate, held once. See `_royal_only` for why the import runs this +# way round. `main.py` already imports both modules, so this adds no load. +import automation_engine as engine + +router = APIRouter(prefix="/api/v1") + +#: Cache the Odoo follow-up pull briefly. The Streamlit page used `@st.cache_data(ttl=1800)`; the +#: list moves slowly (it is a dunning worklist, not a live feed) and the pull is a multi-model read. +_TTL = 1800 +_cache = {"at": 0.0, "rows": None} + + +def _cs(): + import modules.collections_send as cs + return cs + + +def _royal_only(session: Session) -> Session: + """⛔ See the module docstring, guardrail 3. The send client is env-credentialed, so it is + tenant #0's and only tenant #0's. Refuse for anyone else rather than send as the wrong company. + + Keyed on the runtime, never on a request field: a tenant is a property of the SESSION. + + ⭐⭐ WAVE 35 · T35 / CONTRACT C8 — THE TEST ITSELF NOW LIVES IN ONE PLACE. Statements became an + agent step this wave, so the automation engine has to answer the same question at two more + doors (which tenants see the action, which tenants may store it). C8's words are "the predicate + is imported, never re-expressed", and this is the direction that import can run: the engine + imports no route module and no FastAPI, so reaching `_royal_only` FROM it would drag `deps` and + `routes_admin` into a route-free module. The engine holds the boolean; this holds the refusal. + ⚠ WHAT STAYS HERE IS THE HTTP SHAPE, and that is deliberate — a 404 rather than a 403, so the + surface is invisible rather than forbidden. The engine must not know about status codes.""" + if not engine.is_statement_tenant(session.runtime): + raise err(404, "not_found", "statements are not configured for this workspace") + return session + + +def _gate(session: Session = Depends(admin_gate)) -> Session: + return _royal_only(session) + + +def _rows(force=False): + import time + cs = _cs() + if force or _cache["rows"] is None or (time.time() - _cache["at"]) > _TTL: + _cache["rows"] = cs.load_collection_list(cs.Odoo()) + _cache["at"] = time.time() + return _cache["rows"], time.strftime("%Y-%m-%d %H:%M", time.localtime(_cache["at"])) + + +def _public(row): + """Strip the internals the Streamlit grid also hid (`_`-prefixed + partner_id is kept, because + the client needs a stable row identity that is not the display name).""" + return {k: v for k, v in row.items() if not str(k).startswith("_")} + + +@router.get("/admin/statements") +def statements(refresh: int = 0, session: Session = Depends(_gate)): + """The worklist + everything the sender UI needs to render itself honestly.""" + cs = _cs() + try: + rows, loaded_at = _rows(force=bool(refresh)) + except Exception as e: + raise err(502, "odoo_unavailable", f"could not load the collection list: {str(e)[:200]}") + return { + "rows": [_public(r) for r in rows], + "loadedAt": loaded_at, + # The guardrail is reported, never decided, here — the data layer owns it. + "safeMode": bool(cs.SAFE_MODE), + "safeRecipients": sorted(cs.SAFE_RECIPIENTS), + "sender": {"name": cs.SENDER_NAME, "email": cs.SENDER_EMAIL, + "replyTo": cs.REPLY_TO, "company": cs.COMPANY}, + "templates": {"subject": cs.DEFAULT_SUBJECT, "intro": cs.DEFAULT_INTRO, + "footer": cs.DEFAULT_FOOTER}, + # W35-T35: ONE literal, read from the engine, so the sender's worklist and a statements + # agent's tier filter cannot come to mean different things. + "tiers": list(engine.STATEMENT_TIERS), + } + + +def _find(rows, customer): + return next((r for r in rows if r.get("Customer") == customer), None) + + +@router.post("/admin/statements/preview") +def preview(body: dict = Body(default=None), session: Session = Depends(_gate)): + """Render ONE customer's statement exactly as the send path would.""" + cs = _cs() + body = body or {} + rows, _ = _rows() + row = _find(rows, body.get("customer")) + if row is None: + raise err(404, "not_found", "no such customer on the collection list") + t = body.get("templates") or {} + import datetime as dt + month = dt.date.today().strftime("%B %Y") + subject = (t.get("subject") or cs.DEFAULT_SUBJECT) + try: + subject = subject.format(customer=row["Customer"], company=cs.COMPANY, month=month) + except Exception: # noqa: BLE001 + # An unknown placeholder is the user's typo, not a 500. Show the template verbatim so they + # can see what they typed rather than getting an opaque error. + # ⛔ W36-T42: THIS CAUGHT THREE OF THE WAYS `str.format` FAILS AND THERE ARE MORE. + # `{customer` is a ValueError, `{customer.x}` an AttributeError, `{customer:%Y}` a + # TypeError, and every one of them is the same user mistake this branch was written for. + # Naming a subset of the exception types turns a typo in the OTHER half into a 500 on the + # one screen somebody opens to avoid mailing 200 people the wrong thing. + pass + return { + "html": cs.render_statement_html(row, t.get("intro") or cs.DEFAULT_INTRO, + t.get("footer") or cs.DEFAULT_FOOTER), + "to": row.get("Email") or "", + "subject": subject, + } + + +@router.post("/admin/statements/send") +def send(body: dict = Body(default=None), session: Session = Depends(_gate)): + """Queue statements. Returns per-customer outcomes — NEVER a bare count. + + `overrideTo` is the test-send path: one customer, one address. SAFE_MODE still applies (the + data layer refuses an address outside the allow-list), which is why the route does not check it. + """ + cs = _cs() + body = body or {} + names = [str(n) for n in (body.get("customers") or []) if str(n).strip()] + if not names: + raise err(400, "bad_request", "name at least one customer") + # ⛔⛔ W36-T42 / D-299 — A DECLARED TEST SEND WITH NO ADDRESS REFUSES. IT DOES NOT BECOME A + # REAL ONE. This read `(body.get("overrideTo") or "").strip() or None`, so a caller who sent + # `overrideTo: ""` (the field left blank, the state not yet typed into, a trimmed-away space) + # got a REAL statement mailed to the debtor's own address, reported back as `test: false`. On + # the one route in this product that is allowed to write to Odoo, the difference between a + # rehearsal and mailing a live customer was one empty string. + # + # ⭐ THE DISCRIMINATOR IS THE CALLER'S DECLARATION, NOT THE VALUE. A body with no `overrideTo` + # key and no `test` flag is the PRODUCTION send, and refusing that would delete the feature + # D-299 exists to preserve. What can be refused is a caller who SAID this is a test: the key + # being present, or `test: true`, is that statement, and an empty address beside it is the + # mistake. So both spellings of "absent" are covered: the key present and blank, and `test` + # asserted with no key at all. + # + # ⚠ THIS IS THE SECOND OF THREE WALLS AND THE ONLY ONE A PAYLOAD CANNOT ROUTE AROUND. + # `automationApi.testSendStatement` refuses an empty address before the request is built, and + # SAFE_MODE refuses an address outside the allow-list inside `queue_statement`. The client + # wall is bypassable by construction (anything can POST); this one is not. + declared_test = ("overrideTo" in body) or bool(body.get("test")) + override = str(body.get("overrideTo") or "").strip() + if declared_test and not override: + raise err(400, "no_override_address", + "a test send needs the address to send the test to. Without one this would " + "mail the customer's own address, which is the opposite of a test") + override = override or None + if override and len(names) != 1: + raise err(400, "bad_request", "a test send takes exactly one customer") + t = body.get("templates") or {} + rows, _ = _rows() + + sent, failed, skipped = [], [], [] + for name in names: + row = _find(rows, name) + if row is None: + failed.append({"customer": name, "error": "not on the current collection list"}) + continue + if not override and not row.get("Email"): + # The Streamlit page warned and skipped these. Reporting them SEPARATELY from failures + # keeps "we could not" distinct from "there was nowhere to send". + skipped.append({"customer": name, "reason": "no email address on the customer record"}) + continue + try: + mid = cs.queue_statement(cs.Odoo(), row, t.get("subject") or cs.DEFAULT_SUBJECT, + t.get("intro") or cs.DEFAULT_INTRO, + t.get("footer") or cs.DEFAULT_FOOTER, override_to=override) + sent.append({"customer": name, "to": override or row.get("Email"), "mailId": mid}) + except Exception as e: + # SafeModeBlocked lands here too, and that is correct: to the caller a guardrail refusal + # and an Odoo error are both "this one did not go", each with its own honest message. + failed.append({"customer": name, "error": str(e)[:200]}) + return {"sent": sent, "failed": failed, "skipped": skipped, + "safeMode": bool(cs.SAFE_MODE), "test": bool(override)} + + +# --------------------------------------------------------------------------------------------- +# WAVE 35 · T36 / CONTRACT C8 / OWNER RULING R10 — THE AGENT'S PARKED BATCH. +# +# ⛔⛔ THE AGENT ASSEMBLES; A PERSON CLICKS SEND. `automation_engine.run_statements` renders a batch +# and parks it on the automation definition, and it imports no mail path at all. THIS is the only +# door that releases one, and it is deliberately built out of the parts already here: +# · `_gate` — `admin_gate` THEN `_royal_only`, byte-for-byte the dependency the other three +# endpoints use. Not a copy of the checks: the same object. +# · `cs.queue_statement` — the same call `send()` above makes, so SAFE_MODE's allow-list refusal +# happens in the DATA LAYER, where no route, payload or UI can reach around it. +# ⇒ Nothing about the guardrails is re-expressed here, which is what makes "unchanged" checkable. +# +# ⚠ WHY THE BATCH IS RE-RENDERED FROM THE LIVE WORKLIST RATHER THAN SENT AS STORED. The parked +# `html` is what a person APPROVED and is what the review screen shows; but a balance can move +# between the parking and the click, and mailing a figure we know to be stale is worse than mailing +# a fresh one. So the parked batch decides WHO and WITH WHAT WORDS, and the live row decides the +# NUMBERS — the same split `preview` already makes. A customer who has left the worklist entirely +# (they paid) is reported as skipped rather than invoiced. + +@router.get("/admin/statements/agent/{auto_id}") +def agent_batch(auto_id: str, session: Session = Depends(_gate)): + """What is parked and waiting for a click, for ONE agent. `null` when nothing is.""" + defn = (engine.all_definitions(session.runtime) or {}).get(str(auto_id)) or {} + pending = defn.get("pendingStatements") + if not isinstance(pending, dict): + return {"pending": None} + # ⚠ The rendered `html` is NOT returned in the list — a 200-statement batch of rendered mail is + # megabytes, and the review screen needs the count and the names to decide. `preview` already + # renders ONE on demand. + return {"pending": { + "ts": pending.get("ts"), "count": int(pending.get("count") or 0), + "notes": list(pending.get("notes") or []), + "items": [{k: v for k, v in it.items() if k != "html"} + for it in (pending.get("items") or [])], + "safeMode": bool(_cs().SAFE_MODE)}} + + +@router.post("/admin/statements/agent/{auto_id}/send") +def agent_send(auto_id: str, body: dict = Body(default=None), + session: Session = Depends(_gate)): + """Release a parked batch. THE CLICK R10 REQUIRES — nothing else in the system calls this. + + ⛔ IT REFUSES AN EMPTY OR MISSING BATCH rather than answering 200 with nothing sent: a send + that reports success and mails nobody is the `view_upsert` failure mode (200 OK, zero writes) + arriving on the one route in this system that talks to a mail server. + """ + cs = _cs() + defn = (engine.all_definitions(session.runtime) or {}).get(str(auto_id)) or {} + pending = defn.get("pendingStatements") + items = list((pending or {}).get("items") or []) if isinstance(pending, dict) else [] + if not items: + raise err(404, "no_batch", "there is nothing parked for this agent to send") + # ⚠ An explicit subset is allowed (a person unticking a customer on the review screen) but it + # may only NARROW the parked batch. A name that was never parked cannot be introduced by the + # payload, or the review stops being what authorised the send. + want = {str(n) for n in (body or {}).get("customers") or []} + if want: + items = [it for it in items if str(it.get("customer")) in want] + if not items: + raise err(400, "bad_request", "none of those customers are in the parked batch") + rows, _ = _rows() + sent, failed, skipped = [], [], [] + for it in items: + name = str(it.get("customer") or "") + row = _find(rows, name) + if row is None: + skipped.append({"customer": name, + "reason": "no longer on the collection list, so nothing is owed"}) + continue + try: + # ⛔ THE SAME DATA-LAYER CALL `send()` MAKES. SafeModeBlocked is raised INSIDE + # `queue_statement`, so the guardrail cannot be argued with from here. + mid = cs.queue_statement(cs.Odoo(), row, + str(it.get("subject") or "") or cs.DEFAULT_SUBJECT, + cs.DEFAULT_INTRO, cs.DEFAULT_FOOTER) + sent.append({"customer": name, "to": row.get("Email"), "mailId": mid}) + except Exception as e: # noqa: BLE001 + failed.append({"customer": name, "error": str(e)[:200]}) + # ⚠ CLEARED ONLY WHEN NOTHING IS LEFT TO RETRY. A batch dropped while some of it failed would + # lose the list of who still needs a statement, and nobody would know to look. + if sent and not failed: + engine.clear_statements(session.runtime, auto_id) + return {"sent": sent, "failed": failed, "skipped": skipped, + "safeMode": bool(cs.SAFE_MODE), "cleared": bool(sent and not failed)} diff --git a/api/routes_tables.py b/api/routes_tables.py index e15412f5ac6cc1215e9cdbb0750e74eab59c1cf2..79307c310d70c1d7c4d8d236bed5e69dfe917078 100644 --- a/api/routes_tables.py +++ b/api/routes_tables.py @@ -1,2197 +1,2197 @@ -"""routes_tables.py — USER TABLES over the wire (wave 18, contract C3-UT). - -The runtime-database primitive: `core/user_tables.py` (wave-9 C6, host-only until now) served -through the SAME grid machinery every other topic rides — `table_store` for the per-user -workspace strata, `aios_grid.workspace_wire` for the wire shape, `core.grid_events` for every -durable write except rows. Rows are the one genuinely new channel: the events seam has no row -event types (the user_tables docstring's `row_add` gate was described, never built), so row -add/delete/patch are REST endpoints here, walled by `user_tables.is_user_table` + -`user_tables.may_open` — a connector-backed table can never accept an invented row. - -TENANCY: every store touch goes through `session.runtime` (the tenant's store handle), so a -Nurilab admin's tables live under Nurilab's prefix/repo, and the isolation gate's proof #3 -covers them for free. - -VISIBILITY is `user_tables.may_open` — creator, admin, or a `core.shares` grant, fail-closed, -applied in `_defn_or_refuse` before any payload is built. - -⭐⭐ WAVE 36 (W36-T21 / OWNER RULING R6) — AND IT IS NO LONGER THE WHOLE WALL, WHICH IS THE POINT -OF THE TICKET. That paragraph used to end *"the per-table wall is the whole wall"*, and it was -true: a `ut_*` database was a BINARY door, so an admin could hand somebody all 31,418 rows of -`ut_odoo_invoices` or none of them, while `customer_data` had per-user row filters and hidden -fields. `perms.py`'s own docstring booked the fix and warned what half a fix looks like — *"a -stored `ut_*` wall would be INERT: the editor would say DENY, the table routes would keep serving, -and nothing anywhere would say so."* - -So THREE things are now true of every row this file serves, and each has one place: - * `perm_scope.may_read` — IF: admin, then an explicit stored `access: false`, then `may_open` - UNCHANGED. Composed, never merged. - * `scoped_pool`/`scoped_pids` — WHICH ROWS: the permanent filter, applied BEFORE `pids` is - taken, exactly where `routes_customers.grid_assembly` applies it. - * `ut_assembly` — WHICH FIELDS: the transitive hidden closure, applied AFTER - `workspace_wire`, on BOTH wires (the field list and the row payload). - -⛔ AND A DOOR THAT CANNOT APPLY ANY OF IT REFUSES RATHER THAN SERVING THE LOT — `_defn_or_refuse`'s -`scope_applied` flag, which defaults to fail-closed precisely because the one caller outside this -file cannot pass it. -""" -import threading -import json -import time - -from fastapi import Body, Depends -from fastapi import APIRouter - -from deps import Session, err, require_session - -router = APIRouter(prefix="/api/v1") - - -def _ut(): - import core.user_tables as user_tables - return user_tables - - -def _ops(session, table_key, st=None): - """This database's per-user workspace store, bound to the tenant. - - ⭐⭐ W36-T24 / D-214 — `st` LETS ONE ASSEMBLY LEND ITS OWN SNAPSHOT, AND THE DOCUMENT IT SAVES - IS `object_shares`, NOT THE WORKSPACE. MEASURED with a call-counting probe over `ut_assembly` - (D-214's own exit condition): one assembly took **5 whole-document reads for an admin and 6 - for a shared, row-scoped user** — `object_shares` TWICE (three times for a non-creator), - `user_tables` once, and `_table_workspace` twice. - - The `object_shares` repeats come from `grid_events._granted_views` and `_granted_folders`, - which ask `shares.shared_with` for the `view` and the `folder` kind separately, each reaching - the store through `tops.st` — this handle. `user_tables.lend` already memoises exactly those - two buckets for one pass (`_LENDABLE`), so handing the ops object a lend collapses them - without touching `core/shares.py`'s semantics or `grid_events`, which is in no wave-36 fence. - - ⛔ WHY IT IS SAFE ON A PATH THAT ALSO WRITES. `_Lent` serves ONLY `user_tables` and - `object_shares`; `_table_workspace` is not lendable, so every workspace read and write - passes straight through to the runtime, and `__getattr__` forwards `update` regardless. The - assembly performs no `object_shares` write, and `patch_row`'s post-write read-back reads - `session.runtime` directly rather than this handle — contract C5's rule, unbroken. - """ - import core.table_store as table_store - return table_store.make(f"{table_key}_table_workspace", - st=st if st is not None else session.runtime) - - -#: WAVE 27 item 2 (contract C2 / amendment A2) — the relation refresh is COALESCED and runs OFF -#: the request path. `{tenant: True}` while a pass is queued; the worker holds the lock. -_REL_LOCK = threading.Lock() -_REL_DIRTY = {} -_REL_RUNNING = {} - - -def _refresh_relations(session): - """Mark this tenant's Links/Rollups stale and refresh them AFTER the response, once. - - ⛔ WHY THIS IS NOT A DIRECT CALL ANY MORE, and the numbers are the argument. The owner's - item 2 was "adding a new record visually takes too long, I need to be able to spam it", and - this function was the largest single cost inside `POST /tables/{key}/rows`: - `engine.refresh_relations` DEEP-COPIES every table and every row in the tenant before it can - decide whether anything needs doing (`automation_engine.py:9383-9388`), and when something - does it commits with `flush="sync"` — a store round trip, i.e. an HF Dataset commit on the - default backend. Every added record paid a whole-tenant snapshot plus a synchronous commit - before the 201 came back. - - ⛔ AND BACKGROUNDING ALONE WOULD HAVE BEEN A WORSE BUG. Twenty rapid adds would queue twenty - whole-tenant snapshots, each one re-reading a store the previous one just wrote — the spam - the item asks us to support is exactly the load that would melt it. So this COALESCES: at - most one pass runs, and at most one is queued behind it. - - ⚠ THE DIRTY FLAG IS THE LOAD-BEARING PART, not the lock. A write landing WHILE a pass is in - flight must still get a pass afterwards, because the in-flight one snapshotted before that - write existed. Without the flag the LAST add in a burst is precisely the one whose rollups - never update — the failure nobody would notice until a total was quietly wrong. - - Eventual consistency is the accepted trade and was already the documented posture: the tick - repairs materialised cells regardless, and answering 503 here would invite the browser to - repeat a mutation that already succeeded. - """ - tenant = str(getattr(session, "tenant", "") or "") - rt = session.runtime - with _REL_LOCK: - _REL_DIRTY[tenant] = True - if _REL_RUNNING.get(tenant): - return # a worker is live; it will see the flag and loop - _REL_RUNNING[tenant] = True - - def _worker(): - import automation_engine as engine - try: - while True: - with _REL_LOCK: - if not _REL_DIRTY.get(tenant): - _REL_RUNNING.pop(tenant, None) - return - _REL_DIRTY.pop(tenant, None) - try: - engine.refresh_relations(rt, log=lambda *_args: None) - except Exception as exc: # noqa: BLE001 - # The source write already landed and the response is already sent. Log and - # let the tick repair it; never retry in a tight loop. - print(f"[tables] relation refresh deferred: {type(exc).__name__}: {exc}") - finally: - # Belt for an unexpected raise on the bookkeeping itself: a tenant left marked - # RUNNING would never refresh again for the life of the process. - with _REL_LOCK: - if _REL_RUNNING.get(tenant) and not _REL_DIRTY.get(tenant): - _REL_RUNNING.pop(tenant, None) - - threading.Thread(target=_worker, daemon=True, - name=f"rel-refresh:{tenant or 'default'}").start() - - -def _defn_or_refuse(session, table_key, st=None, defs_only=False, scope_applied=False): - """The per-table wall: 404 for a key that does not exist, 403 for one this session may not - open. 404-before-403 leaks nothing useful — ut keys are guessable slugs, and 'exists but - not yours' is exactly what may_open is for. - - ⭐⭐ W36-T21 / R6 / CONTRACT C1 — `scope_applied` IS THE HALF THAT MAKES THE WALL NON-INERT, - AND IT DEFAULTS TO FAIL-CLOSED ON PURPOSE. - - `perms.py`'s own docstring described exactly the defect this parameter prevents: *"a stored - `ut_*` wall would be INERT — the editor would say DENY, the table routes would keep serving, - and nothing anywhere would say so."* So a door that will apply C1's row/field wall to what it - is about to serve says so HERE, in writing, and a door that will not is REFUSED for any - principal carrying a wall on this database (`perm_scope.wall_declared`). - - ⛔ THE DEFAULT IS `False` BECAUSE THE ONE CALLER OUTSIDE THIS FILE CANNOT PASS IT. - `routes_odoo_tables`' windowed rows route calls this guard and then builds its own SQL — it - has no way to apply a row filter or a hidden-field closure, and it is in no wave-36 fence. An - opt-OUT default would have left that door silently serving a walled account the whole table, - which is the INERT wall by another route. Opting IN means a door added tomorrow is refused - until somebody has thought about it ([[default-must-pass-its-own-guard]]). - - ⚠ AND IT COSTS NOTHING FOR EVERYBODY WITH NO WALL — which today is every account in every - tenant, because no `ut_*` wall has ever been storable. `wall_declared` is False for an admin - and False for a record with no entry, so this branch cannot fire until an administrator - deliberately stores one. - - ⭐ `st` LETS A CALLER LEND THE SNAPSHOT IT IS ALREADY HOLDING (W31 QA), the same `lend()` the - nav and `/tables` use since W31-T10/T12. The WALL is untouched — the same `may_open`, asked - about the same table — it is simply not re-reading a 28.5 MB document to ask it. - - ⭐⭐ W33-T01 (D-213) — `defs_only=True` MAKES THAT READ A PROJECTION, AND IT IS OPT-IN PER CALL - SITE ON PURPOSE. This wall asks two questions ("does the key exist", "may this session open - it") and neither has ever read a row, but three of its ten callers go on to read `rows` OFF THE - DEFINITION THIS RETURNS — so a blanket swap would turn `scoped_pool`, `scoped_pids` and - `table_footprint` into `KeyError`s on every materialised table. The flag is therefore the - CALLER's claim about what it will do next, not a global setting; each opt-in below is annotated - with why it can make that claim ([[reuse-and-delete-are-hypotheses]]). - - ⛔ **`defs_only` IS FOR READS.** The six write walls (`_records_or_refuse`, `patch_shared_cell`, - `delete_shared_field`, `patch_table`, `delete_table`, `import_rows`) deliberately do NOT pass - it: a projected snapshot must never reach a post-write read-back (contract C5), and the - pre-write lend note below is the same argument one layer down. - - ⚠ **WHAT COMES BACK IS A `store._Projected`, WHICH IS THE EVIDENCE, NOT AN IMPLEMENTATION - DETAIL.** `all_defs` falls back to the whole read on ANY failure, so "the answer was right" - proves nothing about which read produced it. `core.store.is_projected(defn)` is how a gate - asserts the fast path was TAKEN. - """ - ut = _ut() - # ⭐⭐ W31 QA — THE SWEEP, AND IT IS ONE LINE BECAUSE IT IS DONE HERE RATHER THAN PER ROUTE. - # Owner: *"this is just one case, I need you to check and apply the fix everywhere too."* - # This guard has TEN direct call sites (counted 2026-08-14; the note said FOURTEEN, which was - # the route count, not the caller count) and cost TWO whole-document deep copies at every one - # (`get`, then `may_open`) — on tenant #0 that is 2 x 28.5 MB (D-185) to answer two questions - # about ONE table. Lending here fixes every caller at once, including the eight routes the - # sweep enumerated (`PATCH /shared/{pid}` · `DELETE /shared/fields/{k}` · `GET|POST /rows` · - # `POST /rows/import` · `POST|PATCH /fields` · `PATCH /rows/{pid}`). - # ⛔ WHY IT IS SAFE HERE AND WOULD NOT BE IN THE ROUTES: a lend is a PRE-WRITE snapshot. The - # guard runs before any mutation and returns only the DEFINITION, so the snapshot never - # survives to serve a read-back. Blanket-replacing `st=session.runtime` inside the routes - # would hand that stale snapshot to `patch_row`'s and `add_row`'s post-write read-back, which - # is [[refetch-eats-its-own-write]] with the sign flipped — a write that reports the value it - # replaced. The remaining in-route reads are deliberately untouched and booked instead. - if st is None: - st = ut.lend_defs(session.runtime) if defs_only else ut.lend(session.runtime) - defn = ut.get(table_key, st=st) - if not defn: - raise err(404, "unknown_table", "that database does not exist") - # ⭐⭐ W36-T21 — ONE evaluator for the IF question, and it CALLS `may_open` rather than - # replacing it. `perm_scope.may_read` is admin -> an explicit stored `access: false` -> - # `user_tables.may_open`, unmodified. Two questions stay two questions: `may_open` still - # decides IF the database is visible, C1 decides WHICH rows and fields. - import core.perm_scope as perm_scope - if not perm_scope.may_read(session.user, table_key, st=st): - raise err(403, "forbidden", "that database belongs to another user") - if not scope_applied and perm_scope.wall_declared(session.user, table_key): - # R6's second sentence: a limit that cannot be met is a SENTENCE naming the cause and a - # fix, never a short answer — and here the short answer would be the WHOLE database. - raise err(409, "scope_not_applied", - "an administrator has restricted which rows and columns of this database you " - "may see, and this view cannot apply that restriction. Open the database from " - "the navigation, where the restriction is applied, or ask an administrator to " - "remove it") - return defn - - -def _records_or_refuse(session, table_key, st=None): - """The human record-write wall for a database the automation engine owns.""" - # One lend for BOTH questions this wall asks — the definition wall and the record-mode wall — - # so the two are answered from one read instead of two. Same reasoning as `_defn_or_refuse`. - st = st if st is not None else _ut().lend(session.runtime) - # ⭐ W36-T21 — `scope_applied=True` because every row write behind this wall is bounded by a - # SCOPED pid set of its own: `patch_row` refuses a pid outside `ut_assembly`'s `pids`, and - # `delete_row` asks the same question just below. Refusing here instead would make a walled - # database READ-ONLY rather than row-scoped, which is a different product. - defn = _defn_or_refuse(session, table_key, st=st, scope_applied=True) - if not _ut().records_mutable(table_key, st=st): - raise err(403, "records_read_only", - "records in this automation-owned database are read-only. Add Instagram " - "handles in a Profile database and let enrichment populate this database") - return defn - - -#: ⛔ THE PER-USER CANDIDATE WALL IS RETIRED (WAVE 27, DEBT D-72). **THE TENANT IS THE UNIT, NOT -#: THE USER** — one profile is ONE row, and everyone who may open the database sees all of it. -#: -#: WHY IT HAD TO GO, and it is not a preference: wave 26's R4 made the candidate identity -#: `(platform, handle)` and `_merge_candidates` stamps only the FIRST finder, later finders never -#: overwriting. Combined with a wall that then showed a non-admin only `created_by == me`, the -#: two were SILENT DATA LOSS — the second finder's row was merged away into the first finder's, -#: and the wall then hid the survivor from the person who just found it. They searched, they -#: paid, and the screen said nothing arrived. -#: -#: ⚠ AND THE PAIR WAS MUTUALLY MASKING, which is why it stayed green for a wave: the wall named -#: `ut_ig_candidates`, and a READ-ONLY census of all four tenant stores -#: (`ops/w26_candidate_census.py`) proved that table exists in NO tenant — since wave 25's R2 the -#: write target is whatever database the user points the Create-record action at. So the wall -#: governed a table nobody writes, and fixing EITHER half alone would have armed the other -#: ([[defects-that-mask-each-other]]). -#: -#: The register offered two exits and R4 already implied this one. Restoring per-user visibility -#: instead would have required R4's merge to stop crossing users — a bigger change, against the -#: ruling, to bring back a wall that never governed anything real. -#: -#: ⛔ DO NOT RE-ADD THIS BY INFERRING THE RULE FROM A `created_by` COLUMN. Every -#: automation-written table has one (a scraped row says `automation`), so inference would hide -#: every scraped row from every non-admin — the same disappearance defect, one table wide instead -#: of one table narrow. `created_by` survives as W26/R4's informational "Found by" stamp ONLY. - - -def _too_big(): - """`routes_odoo_tables.TooBigToMaterialise`, imported lazily — one name, two policies below.""" - import routes_odoo_tables - return routes_odoo_tables.TooBigToMaterialise - - -def _read_through_rows(table_key, field_keys, rt=None): - """The mirror's rows for one read-through grid, projected to this table's declared columns. - - ⭐⭐ W31-T45 / D-169 — `rt` IS THE SESSION'S TENANT RUNTIME AND IT IS PASSED THROUGH (D's ask, - `mailbox/D.md` D-2). `_defn_or_refuse` above answers *"may this SESSION open this DATABASE"*; - the guard `whole_pool` fires on `rt` answers a DIFFERENT question — *"is the DuckDB file this - process has open THIS TENANT's"* — and R2 gives GTM Lab connected tables in its own document, - which is exactly the shape that satisfies the first wall while failing the second. - `datastore.ro_con()` reads a process-global `DB_PATH` and one Space process serves every - tenant, so the two walls are not substitutes for each other. - - ⛔ ONE FETCH, TWO POLICIES — and the split is the whole of W31-T20. Both `scoped_pool` (which - owes a caller every row) and `scoped_pids` (which owes only the row SET) reach the mirror - through this function, so the pid set the cheap path answers with is IDENTICAL to the pool's - by construction rather than by a second query that agrees today. A pid-only `SELECT` would be - cheaper and would also be a SECOND statement of what a row of this table is - ([[one-question-two-normalizers]]) — the two callers differ in what they do with the REFUSAL, - never in how they ask. - - Raises `TooBigToMaterialise` when the population exceeds one window; the caller decides - whether that is a 409 or an unresolved pid scope. - """ - import routes_odoo_tables - - rows_src = [{k: v for k, v in r.items() if k in field_keys or k == "pid"} - for r in routes_odoo_tables.whole_pool(table_key, rt=rt)] - rows_src.sort(key=lambda r: r["pid"]) - return rows_src - - -#: ⭐⭐ W31-T20 / D-174 — R6's SECOND SENTENCE FOR A PID SCOPE THAT CANNOT BE RESOLVED. -#: -#: Wave 30 un-capped the ROW door and left the WORKSPACE envelope, so `GET /workspace?scope= -#: ut_odoo_gl_lines` answered `409 window_required` six times out of six on the live deploy and the -#: grid painted an ERROR PAGE with a Retry button — a whole shipped feature nobody could open. -#: The cause: an envelope needs no row, but it asked for every one of 963,783 of them to derive a -#: pid set it uses for exactly two things (cohort membership and a shared view's `memberPids`). -#: -#: ⛔ SO THE ENVELOPE STOPS ASKING, AND SAYS SO. An empty pid set is not "no rows" — it is -#: "membership is unresolved on this grid", which is a different claim and has to be made out -#: loud, on the wire, or it is the silent truncation R6 is actually about. Both consequences are -#: fail-closed: a stored cohort's members are reported MISSING by `aios_grid.workspace_wire` -#: rather than silently dropped, and any WRITE that names a pid is refused by `routes_grid`. -_PID_SCOPE_LIMIT = { - "subject": "pids", "effect": "unresolved", - "recommendation": "cohorts and shared-view membership are resolved per page on this grid; " - "filter and read it a window at a time (`/odoo-tables/{key}/rows`), where " - "every total is a SQL count over the whole table", -} - - -def scoped_pool(session: Session, table_key: str, st=None): - """`(pids, rows_src, fields_base, defn)` — THE USER-TABLE WALL, on its own. - - ⭐ W33-T03 (D-214) — `st` LETS A CALLER LEND THE DOCUMENT IT IS ALREADY HOLDING, exactly as - `_defn_or_refuse` has since W31 QA. It is passed straight through to that wall and nowhere - else, so the permission question is answered by the same code against the same document. - ⛔ READ CALLERS ONLY. A lend is a PRE-WRITE snapshot; handing one to a path that writes and then - reads back is [[refetch-eats-its-own-write]] with the sign flipped (contract C5). - - `routes_products.scoped_pool`'s sibling, extracted for the same reason (wave 19, item 12): a - caller that only needs "which rows of this database may this session touch" — record comments - — must ask through `_defn_or_refuse` (404/403, the whole wall) rather than growing a second - idea of what a user table's pool is. - - ⭐⭐ W36-T21 / R6 — THE ROW WALL RUNS HERE NOW, ON EVERY DATABASE, and the position is the - whole of it: BEFORE `pids` is taken. `routes_customers.grid_assembly` says the same thing in - the same words for `customer_data` — everything downstream is bounded by that frozenset - (`allowed_pids` for the workspace, cohort membership, every write door's pid check), so - scoping here means a row this account may not see never enters ANY of them, rather than being - filtered out of one payload and surviving in another. - - ⚠ THE FIELD HALF IS NOT HERE, and that is parity rather than an omission: the hidden closure - must cover the user's own `custom_`/`measure_` columns, which do not exist until - `workspace_wire` has run. `ut_assembly` applies it there, exactly where `grid_assembly` does. - """ - defn = _defn_or_refuse(session, table_key, st=st, scope_applied=True) - fields_base = [dict(f) for f in (defn.get("fields") or [])] - field_keys = {f["key"] for f in fields_base} - # ⭐⭐ W30-T31 / D-87 — A READ-THROUGH DATABASE HAS NO ROWS HERE, SO THEY COME FROM THE MIRROR. - # - # This is the one function that turns "what is stored" into "what this session may see", which - # is exactly why the read-through arm belongs HERE and nowhere else: the rows route, the events - # route, the comments wall and the assembly all reach rows through it, so they all convert - # together or not at all. ⛔ Reading `defn["rows"]` for such a table would find `{}` and serve - # an EMPTY GRID — correct-looking, wrong, and silent. - # ⚠ The wall above has already run. This adds no scope of its own and takes none away. - import core.perm_scope as perm_scope - if not _ut().materialises(table_key, st=session.runtime, defn=defn): - try: - rows_src = _read_through_rows(table_key, field_keys, rt=session.runtime) - except _too_big() as e: - # R6's second sentence: a limit that cannot be met is a SENTENCE, never a short grid. - # ⚠ THE ROWS PATH STILL REFUSES, AND THAT IS CORRECT (W31-T20 changed the ENVELOPE, not - # this): a caller that asked for every row of a 963,783-row grid cannot be served a - # short one. `scoped_pids` below takes the same refusal and answers a different - # question with it, because an envelope needs no row. - raise err(409, "window_required", str(e)) - except RuntimeError as e: - raise err(503, "store_not_ready", str(e)) - rows_src = perm_scope.apply_row_scope(rows_src, session.user, table_key, fields_base) - return frozenset(r["pid"] for r in rows_src), rows_src, fields_base, defn - rows_src = [] - for rid, row in (defn.get("rows") or {}).items(): - if not str(rid).isdigit(): - continue - # WAVE 27 / D-72: no per-row OWNER filter — the row's creator is not a permission. What - # runs below is a different thing entirely: the permanent filter an ADMIN declared for - # this account (W36-T21 / R6), the same one `grid_assembly` has applied to `customer_data` - # since wave 15. A row this session can reach is a row the tenant owns AND the wall admits. - r = {k: v for k, v in (row or {}).items() if k in field_keys} - r["pid"] = int(rid) - rows_src.append(r) - rows_src.sort(key=lambda r: r["pid"]) - # ⭐⭐ W36-T21 — `permits()`, not `matches()`: an unanswerable permanent filter DENIES rather - # than being ignored. Evaluated against the DECLARED contract (`fields_base`), which is what - # `routes_admin._clean_perms` validates a stored filter against, so the two cannot disagree - # about what a column is. - rows_src = perm_scope.apply_row_scope(rows_src, session.user, table_key, fields_base) - return frozenset(r["pid"] for r in rows_src), rows_src, fields_base, defn - - -@router.patch("/tables/{table_key}/shared/{pid}") -def patch_shared_cell(table_key: str, pid: int, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Write a cell into the TENANT-WIDE overlay — the product door `core/shared_overlay.py` has - been waiting for since it shipped (W29-T62, wave 30 T28). - - ⭐ WHY A SEPARATE STRATUM AT ALL, restated because it is the whole feature and it is not - "sharing would be nice": a user-created column and its values live PER USER, so a shared view - filtering on one names a column other accounts do not have — and an unknown column is an - INACTIVE condition in the tri-state engine, which IGNORES it and therefore WIDENS. The buy - list would silently show the whole catalogue to everyone but its author. A column whose value - is the same for every reader is the precondition for editing it at all. - - ⛔ THE WALL IS `_defn_or_refuse`, AND THE STRATUM IS NOT ONE. `shared_overlay` refuses no - reader and no writer by design; "may this session open this surface" is answered HERE, where - the session is. Do not push the question down there. - ⚠ A tenant-wide write is not a private one: every account that may open this database sees it. - That is the point, and it is why this door declares the column too — a value with no - definition is a cell nobody can find. - """ - body = body if isinstance(body, dict) else {} - key = str(body.get("field") or "").strip() - if not key: - raise err(400, "bad_request", "a field key is required") - _defn_or_refuse(session, table_key) - from core import shared_overlay - import core.perm_scope as perm_scope - if not shared_overlay.is_shared(table_key, key, st=session.runtime): - # ⭐⭐ W38-T16 / R7 — A COLUMN IS BORN OWNED AND PRIVATE, AND THIS IS THE PRODUCT CHANGE. - # Until now this door minted a column every account in the tenant could read, with no way - # to say who. `granted` marks it as governed by `core.shares` (see - # `perm_scope.FIELD_GRANT_MARK` for why an explicit marker and not the absence of a grant - # record), and the grant record below claims it for the creator. - # - # ⚠ THE EMPTY ENTRY LIST IS THE POINT, NOT A PLACEHOLDER. `set_grants` keeps a record that - # has an owner and no entries, so "shared with nobody" is STORED and is a different fact - # from "never shared" — the same explicit-resolution shape `perms_v` uses. Without the - # owner the column would be unmanageable: `may_administer` fails closed on an ownerless - # record, so nobody could ever share or re-share it. - # ⚠ COLUMNS THAT PREDATE THIS CARRY NO MARK and stay tenant-wide, unchanged. Reading - # their absence as "granted to nobody" would blank every existing shared column at once. - shared_overlay.put_field(table_key, key, { - "key": key, "label": str(body.get("label") or key), "source": "overlay", - "type": str(body.get("type") or "text"), "shared": True, - perm_scope.FIELD_GRANT_MARK: True, - "createdBy": session.uname}, st=session.runtime) - try: - import core.shares as shares - shares.set_grants("field", shares.field_oid(table_key, key), [], - owner=session.uname, st=session.runtime) - except Exception: # noqa: BLE001 - # ⛔ THE MARK IS ALREADY WRITTEN, SO A FAILED CLAIM FAILS **CLOSED**: the column is - # governed and nobody holds a grant, i.e. only an admin sees it. That is recoverable - # (an admin can share it) and the other order is not — a marked column with a claim - # that landed first and a definition that did not would be a grant on nothing. - pass - try: - # ⚠ `put_cell`, not `put_cells` — this door writes exactly ONE cell, and the singular is - # the API that says so. It delegates to the plural, so both stay reachable through the one - # caller; before this, the singular had no caller at all and `verify_reachability` LENS 2 - # named it (the same lens that found `drop_field` had no door either). - stored = {key: shared_overlay.put_cell(table_key, pid, key, body.get("value"), - st=session.runtime)} - except ValueError as e: - # A non-scalar RAISES in the stratum rather than being dropped; relay it as the answer. - raise err(400, "bad_value", str(e)) - return {"ok": True, "pid": pid, "cells": stored, - "fields": list(shared_overlay.fields(table_key, st=session.runtime))} - - -@router.delete("/tables/{table_key}/shared/fields/{field_key}") -def delete_shared_field(table_key: str, field_key: str, - session: Session = Depends(require_session)): - """Remove a TENANT-WIDE column and every value in it. - - ⛔ WHY THIS EXISTS AT ALL, said plainly: W30-T28 shipped the door that CREATES a shared column - and none that removes one, so a column anybody added was permanent for the whole tenant. The - reachability gate found it from the other end — `shared_overlay.drop_field` was complete, - correct, gated, and callable by nothing but its own gate ([[reachable-is-not-the-same-as-built]]). - - ⛔ AND THIS ONE IS CREATOR-OR-ADMIN, WHICH THE WRITE DOOR IS NOT. Writing a cell changes a - value; dropping the column deletes that value for EVERY account at once, so it is the - destructive-op wall this repo already uses for a database delete — not `editRole`, which - governs renaming and is not a value wall ([[schema-role-is-not-a-value-wall]]). - ⚠ `createdBy` is stamped by the write door above; a column stored before that stamp existed - is admin-only, which is the safe direction. - """ - _defn_or_refuse(session, table_key) - from core import shared_overlay - defn = (shared_overlay.fields(table_key, st=session.runtime) or {}).get(str(field_key)) - if not defn: - raise err(404, "unknown_field", "that column is not a shared column on this database") - owner = str(defn.get("createdBy") or "") - if not session.admin and owner != session.uname: - raise err(403, "forbidden", - f"a tenant-wide column can be removed by its creator or an admin. This one " - f"was added by {owner or 'somebody else'}, and dropping it would delete the " - f"value for every account") - dropped = shared_overlay.drop_field(table_key, str(field_key), st=session.runtime) - # ⭐⭐ W38-T16 — THE GRANTS DIE WITH THE COLUMN, which is `shares.drop_objects`' whole reason - # for existing (wave 21, C3): a deleted object's grants would otherwise serve a ghost id into - # every receiver's "Shared with me" forever, and the ghost 404s on open. It matters twice as - # much for a field, because `drop_field` scrubs the CELLS so the key can be re-used — and a - # surviving grant record would silently re-arm on the next column that took the name. - # ⚠ R8 / D-172 IS UNCHANGED ABOVE: delete is creator-or-admin. This ticket did not widen it. - try: - import core.shares as shares - shares.drop_objects([("field", shares.field_oid(table_key, str(field_key)))], - st=session.runtime) - except Exception: # noqa: BLE001 - pass - return {"ok": True, "dropped": bool(dropped), - "fields": list(shared_overlay.fields(table_key, st=session.runtime))} - - -def scoped_pids(session: Session, table_key: str, limits=None, st=None): - """`(pids, fields_base, defn)` — the SAME wall and the SAME row set as `scoped_pool`, without - building a row. - - ⭐⭐ WAVE 30 / W30-T30 — THIS IS WHY ONE HIDE-FIELDS CHECKBOX WAS EXPENSIVE. A view write - (`view_upsert`) reaches `grid_events_route`, which built a FULL assembly purely to validate - it: `scoped_pool` allocates a fresh dict per row and then sorts them — ~33k order rows, on - every toggle — and the six keys the events route actually reads from that assembly - (`fields`, `pids`, `measures`, `measure_sets`, `lists`, `views`) contain no row at all. - `rows_src` was computed and discarded. - - ⛔ THE PID SET IS IDENTICAL, NOT MERELY EQUIVALENT, and that is the whole safety argument: - `scoped_pool` derives its pids as `frozenset(r["pid"] for r in rows_src)` over exactly the - row ids that pass `str(rid).isdigit()`, which is this comprehension with a dict build in the - middle. The row WALL is unchanged — a narrower or wider set here would be a permission - change, and this is a performance change. - - ⚠ It does NOT make the write cheap on its own: `_defn_or_refuse` still costs a whole-document - read, which is D-87 and W30-T31. This removes the row pass. - ⭐ CORRECTED 2026-08-14 (W33-T01): that sentence said **two** deep copies (`ut.get` then - `may_open`) and had been stale since W31 QA taught the wall to `lend()` — the two questions - have shared ONE read since `routes_tables.py`'s lend line. And as of this ticket the read is a - PROJECTION on the read-through arm, so the sentence is now true only of the materialised one. - Booked because a stale performance note is how a wave re-fixes something twice - ([[stale-baseline-unreadable-deltas]]). - - ⭐⭐ W31-T20 / D-174 — `limits` IS AN OUT-PARAMETER, AND IT IS THE POINT OF THE TICKET. Pass a - list and this function APPENDS R6's sentence to it when the pid set could not be resolved (a - read-through grid whose population exceeds one window). The pid set is then EMPTY, and every - consumer of an empty pid set is fail-closed — but "fail-closed and unannounced" is exactly the - silent limit R6's second sentence forbids, so a caller that renders an envelope or admits a - write is expected to carry the sentence through. Omitting the list means the caller accepts an - unannounced empty scope, which is only ever right for a caller that does not use the pids. - """ - # ⭐⭐ W33-T01 / D-213 — THE DATABASE-SWITCH PATH, AND IT STARTS ON A PROJECTION. - # - # `GET /workspace?scope=` reaches here through `ut_assembly(with_rows=False)` - # (`routes_grid.py`'s ut_ branch), which is what a person is waiting for when they click a - # database in the nav flyout: 1.8-7.3 s live for a 3-6 KB payload, of which one whole-document - # read is ~703 ms warm and 20.6 s cold. This function reads `fields` and (below) `readThrough` - # off the definition — no row — so the WALL can be answered from the 0.1% projection. - # - # ⛔ IT IS THE TRAP ON THIS BOARD, SO IT IS SAID TWICE: this function is NAMED and DOCUMENTED - # as the rows-free twin of `scoped_pool` and the materialised arm below still reads `rows`. - # The opt-in is therefore CONDITIONAL, and the condition is `materialises`, which reads - # `readThrough` — a definition key, safe under the projection, and already lent the defn so it - # costs no read of its own. - import core.perm_scope as perm_scope - # ⭐ W36-T24 / D-214 — `st` LETS THE ASSEMBLY LEND ITS OWN PASS, exactly as `scoped_pool` has - # since W33-T03. ⛔ It must be a PROJECTED lend (`lend_defs`), not a whole one: the saving - # D-213 bought on the database-switch path is that this wall answers from the 0.1% document, - # and handing it `lend()` would quietly take that back while looking like an optimisation. - defn = _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True) - fields_base = [dict(f) for f in (defn.get("fields") or [])] - # ⚠ W30-T31: on a read-through database the stored `rows` is `{}` by construction, so the - # comprehension below would answer an EMPTY pid set — and the promise this function makes is - # that its set is IDENTICAL to `scoped_pool`'s, not merely cheaper. It reaches the mirror - # through the SAME fetch that function uses rather than growing a second idea of the row set; - # the saving W30-T30 bought stays on every materialised table, which is all of the big ones. - if not _ut().materialises(table_key, st=session.runtime, defn=defn): - try: - rows = _read_through_rows(table_key, {f["key"] for f in fields_base}, - rt=session.runtime) - rows = perm_scope.apply_row_scope(rows, session.user, table_key, fields_base) - return frozenset(r["pid"] for r in rows), fields_base, defn - except _too_big() as e: - # ⛔ THE REFUSAL BECOMES AN ANSWER HERE, WHICH IT MUST NOT ON THE ROWS PATH. Turning - # this into a 409 is what made both line grids unopenable: the envelope was refused - # over rows it never renders. The scope is empty and SAID to be empty. - if limits is not None: - limits.append({**_PID_SCOPE_LIMIT, "cause": str(e)}) - return frozenset(), fields_base, defn - except RuntimeError as e: - raise err(503, "store_not_ready", str(e)) - # ⛔⛔ MATERIALISED: THE PID SET *IS* `rows`, SO THIS ARM TAKES THE WHOLE READ — the projection - # above cannot serve it and would raise rather than answer `{}` (that is the whole design of - # `_Projected`). The wall has already passed on the projected document, so this re-reads the - # DEFINITION and does not re-ask `may_open`: re-walling would be a second, differently-shaped - # answer to a question already answered, which is how two ideas of ownership got into this file - # once before (see `may_open`'s own note in `core/user_tables.py`). - # - # ⚠ THE HONEST COST, STATED RATHER THAN BURIED: a materialised table now pays the projection - # PLUS the whole read — ~1.4 ms on top of ~703 ms on tenant #0, i.e. 0.2%. The pid set, the - # wall and the returned shape are byte-for-byte what they were; only tenant #0's ten - # read-through databases (every one of them, which is why the switch was slow) skip the big - # read entirely. - whole = _ut().get(table_key, st=session.runtime) - if whole is None: - # Between the wall and here the table was deleted by another request. Same refusal the - # wall gives, rather than an empty pid set nobody can distinguish from an empty table. - raise err(404, "unknown_table", "that database does not exist") - # ⭐⭐ W36-T21 — AND THE ROW WALL, WHICH IS WHY THIS ARM CAN NO LONGER ALWAYS SKIP THE ROWS. - # - # ⛔ THE PROMISE THIS FUNCTION MAKES IS THAT ITS PID SET IS **IDENTICAL** TO `scoped_pool`'s, - # not merely cheaper. `scoped_pool` now narrows its rows by the permanent filter before taking - # pids, so a set built here from the raw row ids would be WIDER — and every consumer of these - # pids (the workspace envelope, cohort membership, `patch_row`'s scope check) would admit rows - # the read door refuses. That is two ideas of one row set, which is the exact defect class - # `may_open`'s own wave-20 note records ([[one-question-two-normalizers]]). - # - # ⭐ AND W30-T30's SAVING SURVIVES FOR EVERYBODY IT WAS FOR. `row_scope_applies` is False for - # an admin and for every record with no declared filter, which is every account in every - # tenant today — those callers take the id comprehension exactly as before and build no row. - # Only a principal an administrator has actually row-scoped pays the pass, and for them the - # alternative is not "cheaper" but "wrong". - rows = whole.get("rows") or {} - if not perm_scope.row_scope_applies(session.user, table_key): - return frozenset(int(rid) for rid in rows if str(rid).isdigit()), fields_base, whole - keys = {f["key"] for f in fields_base if f.get("key")} - scoped = perm_scope.apply_row_scope( - [{**{k: v for k, v in (row or {}).items() if k in keys}, "pid": int(rid)} - for rid, row in rows.items() if str(rid).isdigit()], - session.user, table_key, fields_base) - return frozenset(r["pid"] for r in scoped), fields_base, whole - - -def ut_write_ctx(session: Session, table_key: str): - """The g-dict a WRITE needs — same keys as `ut_assembly`, no rows. - - Returns the six keys `routes_grid.grid_events_route` reads, so the events route consumes this - or a full assembly interchangeably. `rows_src` is `[]` on purpose rather than absent: a caller - that starts needing rows should fail on an empty list it can see, not on a KeyError. - """ - import aios_grid - from core import grid_events - - limits = [] - # ⭐ W36-T24 / D-214 — ONE lend for the whole pass, so the wall and the grant legs stop - # reading `object_shares` once each. Projected, because this ctx reads no row either. - lent = _ut().lend_defs(session.runtime) - pids, fields_base, defn = scoped_pids(session, table_key, limits=limits, st=lent) - # ⭐⭐ W36-T21 — THE FIELD WALL ON THE **WRITE** CTX, and it is not a copy of the read one. - # `grid_events` refuses a hidden key by asking `ctx.hidden_keys` (`grid_events.py:1802` and - # `:2021`); this ctx passed `frozenset()`, so a hidden column was hidden on the READ and fully - # writable on the EVENTS transport — a wall on one wire and not the other is the shape - # `strip_row`'s own note warns about, with the sign flipped. - hidden = _ut_hidden(session, table_key, fields_base, st=lent) - ctx = grid_events.EventCtx( - uname=session.uname, allowed_pids=pids, fields=[], hidden_keys=hidden, - admin=session.admin, fallback_ws=None, seen_ids={}, st=session.runtime, - scope_key=table_key, table=_ops(session, table_key, st=lent)) - ws = grid_events.table_workspace(ctx, allowed_pids=pids, consume_corrections=False) - workspace, fields, views, lists = aios_grid.workspace_wire( - ws, session.uname, set(pids), defs={}, scope_key=table_key, storage_key="", - fields_base=fields_base) - # ⭐⭐ W38-T16 — THE SHARED COLUMNS JOIN THE **WRITE** CTX'S CONTRACT TOO, and for the reason - # W36-T21 gave one wall over: `grid_events` refuses a key by asking `ctx.hidden_keys`, so a - # column missing from this list is a column the events transport does not know it must - # refuse. The read door narrowing a name the write door has never heard of is the same - # one-wire wall with the sign flipped. - fields = _ut_shared_fields(session, table_key, fields) - fields, _rows, hidden = _ut_field_wall(session, table_key, fields, [], st=lent) - return {"rows_src": [], "pids": pids, "ws": ws, "workspace": workspace, - "fields": fields, "views": views, "lists": lists, "hidden": hidden, - "derived": aios_grid.cohort_cells(lists), - # ⭐ W37-T12 — the OFFER, because `routes_grid` reads `g["measures"]` as the ADMISSION - # list for `clean_measure_field`: without it a Metric create on this surface is - # refused `measure_not_offered` even though the read door offers the kind. - # ⛔ THE CELLS ARE DELIBERATELY ABSENT HERE. This ctx validates a write and - # materialises no rows at all (W30-T30 removed a full table build from the write - # path); computing measure values would put that work straight back. `measure_sets` - # stays empty for the same reason — a write is not a render. - # ⭐ AND THE ASYMMETRY IS SAFE, WHICH IS NOT OBVIOUS AND WAS CHECKED RATHER THAN - # ASSUMED. `routes_grid._ctx` derives TWO things from this dict and they have - # different jobs: `measure_keys` (from `measures`) is the ADMISSION list - # `clean_filter_tree` uses — non-empty here, so a saved measure CONDITION survives the - # write, which is the failure that comment records for the customer path. Empty - # `resolved_ids` (from `measure_sets`) only makes `view_upsert` answer "rerender", - # so the next `/workspace` — which DOES compute the sets, in `ut_assembly` — resolves - # it. Conservative in the right direction: one extra repaint, never a dropped filter. - # ⭐⭐ W38-T19 — THE CAPABILITY IS APPLIED HERE TOO, and this is the site that makes - # the refusal REAL rather than cosmetic: `routes_grid` reads this list as - # `measure_offer`, and `clean_measure_field` fail-closes `measure_not_offered` over - # an empty one. A picker hidden on the read door with this list still full would be - # a control that only stops the honest. - "measures": ut_measures(table_key, session), - "measure_sets": {}, "today": time.strftime("%Y-%m-%d"), - # ⭐ W31-T20 — the write door reads this to refuse a PID-BEARING event loudly rather - # than letting `allowed_pids` swallow it as a no-op. See `routes_grid`'s ut_ branch. - "limits": limits, "defn": defn} - - -def _ut_hidden(session, table_key, fields, st=None): - """The hidden-field closure for THIS session on THIS database — C1's field half, once. - - ⚠ Named rather than inlined at its four call sites for the reason `may_open`'s own note gives: - four spellings of one wall is how two of them come apart. `perm_scope.hidden_keys` is the ONE - evaluator; this is just the `ut_*` caller's shorthand for it. - - ⭐ W38-T16 — `st` IS THE LEND THE CALLER ALREADY HOLDS, and passing it is free rather than - merely tidy: the closure's field-grant leg resolves against `object_shares`, which is exactly - the bucket `ut_assembly`'s own lend note says the pass is already serving. - """ - import core.perm_scope as perm_scope - return perm_scope.hidden_keys(session.user, table_key, fields, st=st) - - -def _ut_shared_fields(session, table_key, fields): - """`fields` PLUS this database's TENANT-WIDE columns — the read-side merge, W38-T16. - - ⛔⛔ WHY THIS DID NOT EXIST AND WHY THAT WAS A HOLE. `patch_shared_cell` has written into - `__shared` for any `ut_*` database since W30-T28, and only the READ-THROUGH door - (`routes_odoo_tables`) ever merged that stratum back. So a shared column written on a - MATERIALISED user table went into the store and was readable by nobody, through any door — - complete, correct, and invisible ([[reachable-is-not-the-same-as-built]]). This is the other - half of that door. - - ⚠ MERGED BEFORE THE WALL, NEVER AFTER, and `routes_odoo_tables` learned this the same way: - the hidden closure must run on the WHOLE contract, or a formula in a shared column that reads - a hidden one sits outside its reach and carries the hidden value out wearing a second name. - ⚠ A key the definition already declares WINS. A shared column is an addition to a database's - contract, never a redefinition of a column that database already has. - """ - from core import shared_overlay - defs = shared_overlay.fields(table_key, st=session.runtime) or {} - if not defs: - return fields - have = {f.get("key") for f in (fields or ()) if isinstance(f, dict)} - return list(fields or ()) + [dict(f, source="overlay") - for k, f in defs.items() if k not in have] - - -def _ut_shared_cells(session, table_key, pids): - """`{"": {key: value}}` for the TENANT-WIDE stratum, over THIS session's row set. - - ⛔ `pids` IS THE ROW WALL AND IT IS PASSED, NOT DEFAULTED — `shared_overlay.cells` refuses an - "everything" read by signature for exactly this reason, and the set handed in is the one - `scoped_pool`/`scoped_pids` already narrowed. A cell for a row this session may not see - therefore has nothing to attach to. - """ - from core import shared_overlay - try: - return shared_overlay.cells(table_key, list(pids or ()), st=session.runtime) - except Exception: # noqa: BLE001 - return {} - - -def _ut_field_wall(session, table_key, fields, rows_src, st=None): - """`(fields, rows_src, hidden)` with the hidden closure removed from BOTH wires. - - ⭐⭐ W36-T21 — the same three lines `routes_customers.grid_assembly` runs for `customer_data`, - in the same position: AFTER `workspace_wire`, because the closure must cover the user's own - `custom_` and `measure_` columns and those do not exist until it has run. - - ⛔ BOTH WIRES, ALWAYS. `strip_row`'s docstring is the record of why: the field LIST and the - ROW payload are two different wires, and narrowing only the first leaves the value sitting in - the second where anything can read it. A formula (or a rollup) over a hidden column comes out - too — hiding the input while shipping the dependent either leaks the input wearing a derived - column's name or computes a wrong one. - - ⚠ IT TAKES NO `hidden` ARGUMENT, deliberately. The base-level closure the write ctx computed - is a SUBSET of this one by construction — same evaluator, a strictly larger field list — so - accepting it would be a second input that can only ever be redundant, i.e. a parallel code - path with nothing to say ([[one-question-two-normalizers]]). - """ - import core.perm_scope as perm_scope - hide = perm_scope.hidden_keys(session.user, table_key, fields, st=st) - if not hide: - return fields, rows_src, frozenset() - fields = [f for f in fields if f.get("key") not in hide] - rows_src = [perm_scope.strip_row(r, hide) for r in (rows_src or [])] - return fields, rows_src, hide - - -def ut_assembly(session: Session, table_key: str, storage_key: str = "", - consume_corrections: bool = True, with_rows: bool = True, st=None): - """The user-table mirror of `grid_assembly` / `product_assembly` — SAME g-dict keys, so - `/workspace` and the events route consume any of the three interchangeably. - - Honest absence: `measures`/`measure_sets` are EMPTY — `core.measure_resolve` is - customer-grain, so there is nothing to offer over user rows. - - ⭐ WAVE 19 / R9 — `lists` IS NO LONGER EMPTY. "For ANY database new/old": a user table gets - cohorts like every other database, out of its OWN bucket (`ut__cohorts`), holding its - own row ids. The wave-18 refusal was correct while there was one customer-keyed bucket and - wrong the moment the store learned about topics. - - ⭐⭐ W31-T20 / D-174 — `with_rows=False` BUILDS THE ENVELOPE AND NOT THE TABLE, and the - caller that wants it is `/workspace`, which renders no row at all (the grid fetches rows from - `/tables/{key}/rows` or `/odoo-tables/{key}/rows` beside it). Two things follow: - * the read-through line grains become OPENABLE — `scoped_pool` refused their envelope over - 963,783 rows nobody was going to look at, which is D-174 in one sentence; - * every materialised `ut_*` database stops allocating a dict per row and sorting them on a - route whose payload has no rows in it — `ut_odoo_orders` was rebuilding 32,826 of them per - database switch (owner item 7). - ⛔ NOTHING IS VALIDATED LESS. `scoped_pids` runs the SAME `_defn_or_refuse` wall and answers - the identical pid set; the flag removes work, never a check — the shape W30-T30 already proved - on the write door. - """ - import aios_grid - from core import grid_events - - limits = [] - # ⭐⭐ W36-T24 / D-214 — **ONE LEND FOR THE WHOLE PASS**, and it is the shape of the lend that - # keeps D-213's saving. The rows arm needs `rows` off the definition, so it lends the whole - # document; the envelope arm reads no row at all and lends the PROJECTION. Either way the same - # object then serves `object_shares` to the grant legs downstream, so the assembly stops - # reading that bucket once per question asked of it. - if st is None: - st = _ut().lend(session.runtime) if with_rows else _ut().lend_defs(session.runtime) - if with_rows: - # ⭐ W33-T03 (D-214): `st` is a caller's LEND, threaded to the wall and nowhere else. Only - # a READ route passes one — see `scoped_pool`'s own note and contract C5. - pids, rows_src, fields_base, defn = scoped_pool(session, table_key, st=st) - else: - pids, fields_base, defn = scoped_pids(session, table_key, limits=limits, st=st) - rows_src = [] - - # ⭐ W36-T24 / D-214 — ONE lend for the whole pass. A caller that already holds one passes it - # as `st`; otherwise this assembly takes its own. Only `user_tables` and `object_shares` are - # served from it, and the assembly writes neither. - ops = _ops(session, table_key, st=st) - # ⭐⭐ W36-T21 — the WRITE half of the field wall. `frozenset()` here meant a column an - # administrator had hidden was still writable through the events transport. - hidden = _ut_hidden(session, table_key, fields_base, st=st) - ctx = grid_events.EventCtx( - uname=session.uname, allowed_pids=pids, fields=[], hidden_keys=hidden, - admin=session.admin, fallback_ws=None, seen_ids={}, - # R6b (D-16): the tenant handle rides every ctx this layer builds, not just the ones - # that happen to carry a scoped `table`. - st=session.runtime, - scope_key=table_key, table=ops) - ws = grid_events.table_workspace(ctx, allowed_pids=pids, - consume_corrections=consume_corrections) - workspace, fields, views, lists = aios_grid.workspace_wire( - ws, session.uname, set(pids), defs={}, scope_key=table_key, - storage_key=storage_key, fields_base=fields_base) - # ⭐⭐ W38-T16 — THE TENANT-WIDE STRATUM, ON A MATERIALISED GRID. Merged HERE, between - # `workspace_wire` and the wall, which is `routes_odoo_tables`' own position for the same two - # lines and for the same reason: the closure must be recomputed on the MERGED contract or a - # formula in a shared column reaches past it. - fields = _ut_shared_fields(session, table_key, fields) - # ⚠ THE CELLS COST A READ, SO THE ENVELOPE ARM DOES NOT PAY IT. `with_rows=False` renders no - # row at all (D-174), and `cells()` over the whole pid set would be a scoped read whose - # result nothing consumes. The COLUMNS still merge above: `/workspace` needs the contract. - shared_cells = _ut_shared_cells(session, table_key, pids) if with_rows else {} - # ⭐⭐ W36-T21 / R6 — the READ half, in `grid_assembly`'s own position: after `workspace_wire`, - # so the closure covers this user's `custom_` and `measure_` columns too. - fields, rows_src, hidden = _ut_field_wall(session, table_key, fields, rows_src, st=st) - if hidden and shared_cells: - # ⛔ BOTH WIRES, AND THE SHARED STRATUM IS A THIRD ONE. `strip_row` above cleaned the - # DEFINITION rows; these cells arrived by a different door and one narrowing cannot speak - # for both — the same sentence `routes_odoo_tables` writes over its own `overlays` dict. - shared_cells = {pid: {k: v for k, v in cells.items() if k not in hidden} - for pid, cells in shared_cells.items()} - - # ⭐⭐ W37-T12 / owner item 4 (R1) — `measures` IS NO LONGER UNCONDITIONALLY EMPTY. A database - # whose entity topic declares a `measures:` binding (today `ut_odoo_agents`) serves a real - # lookback catalogue AND the cells to go with it; every other user table still serves `[]`, - # which is the honest answer for a hand-typed one rather than a descope. - today = time.strftime("%Y-%m-%d") - measures = ut_measures(table_key, session) - derived = aios_grid.cohort_cells(lists) # R9: this table's own lists - for _pid, _cells in _ut_measure_cells(table_key, fields, pids, today, measures, - session.runtime.measure_memo).items(): - derived.setdefault(_pid, {}).update(_cells) - - return {"rows_src": rows_src, "pids": pids, "ws": ws, "workspace": workspace, - "fields": fields, "views": views, "lists": lists, "hidden": hidden, - "derived": derived, - # ⭐ W38-T16 — ALWAYS PRESENT, `{}` when this database shares nothing. A key a - # consumer has to test for is a key a consumer forgets to test for, and the consumer - # here (`table_rows`) is layering strata in a fixed order. - "shared_cells": shared_cells, - "measures": measures, - "measure_sets": _ut_measure_sets(table_key, views, measures, pids, today), - "today": today, - # ⚠ ALWAYS PRESENT, EMPTY WHEN THERE IS NOTHING TO SAY — a key a consumer has to test - # for is a key a consumer forgets to test for, and this one carries a refusal. - "limits": limits, "defn": defn} - - -def _ut_measure_err(tag, e): - try: - import harness.telemetry as _tel - _tel.error(f"api:{tag}", e) - except Exception: # noqa: BLE001 - pass - - -def ut_measures(table_key, session): - """The lookback-measure OFFER for a user database FOR THIS SESSION, or `[]` — W37-T12 / - owner item 4 (R1), narrowed by the metrics capability in W38-T19. - - ⭐⭐ W38-T19 — `session` IS REQUIRED, NOT DEFAULTED. The function was PRINCIPAL-BLIND, so a - per-user capability had nothing to be consulted by. Defaulting it would leave both existing - call sites compiling and silently ungated, which is the fail-open spelling of the bug rather - than a fix for it — the argument `_module_fields` in `routes_admin` records for its own - `session`. ⛔ AND BOTH CALL SITES MATTER, WHICH IS EASY TO GET HALF RIGHT: `ut_assembly` is - the READ and `ut_write_ctx` is the door `routes_grid` turns into `clean_measure_field`'s - admission set. Gating only the read would take the kind off the picker and leave the CREATE - working for anyone who posts the event. - - ⭐ THE SAME ENGINE THE PRODUCT GRID USES (`semantic.entity_measures`), reached through the - topic that DESCRIBES this database. A table with no entity topic, or a topic with no - `measures:` binding, gets `[]` — which is the honest answer for a hand-typed database and is - what keeps the Metric kind hidden there (`ColumnMenu`'s `offerMeasure={measures.length > 0}`). - - ⛔ NOT A `{table_key: [keys]}` MAP IN THIS FILE. The binding is declared in the model - (`model/topics/odoo_agents.yml`'s `measures:` block) and the topic already names its own grid, - so a list here would be a second definition of one fact — the argument `_rollup_source_offer` - makes about itself one screen down. - """ - import core.perm_scope as perm_scope - if not perm_scope.may_metrics(session.user, table_key): - return [] - try: - from harness import semantic as sem - topic = sem.topic_for_grid(table_key) - return sem.entity_measures(topic) if topic else [] - except Exception as e: # noqa: BLE001 - _ut_measure_err("ut-measure-offer", e) - return [] - - -def _ut_measure_cells(table_key, fields, pids, today, offer, memo, team_id=None): - """`{pid: {field key: value}}` for this database's Metric columns. - - ⭐ THE JOIN NEEDS NO MAP HERE, and that is worth stating because the product grid DOES need - one. `sales_lines.agent` is `rp.agent_id` — a `res.partner` id — and a `ut_odoo_agents` row's - `pid` IS its `res.partner` id (`scoped_pool` sets `pid = int(rid)` and `read_agents` keys - `_id` on the partner id). So the group key and the pid are the same integer. The product grid - keys on `default_code` instead, which is exactly why contract C1 calls the join key a trap: - the right answer differs per database and must be read off the binding, never assumed. - """ - import aios_grid - - mfields = aios_grid.measure_fields_of(fields) - if not mfields or not offer: - return {} - from harness import semantic as sem - from harness import windows as _wn - - topic = sem.topic_for_grid(table_key) - by_key = {m["key"]: m for m in offer} - by_window = {} - for f in mfields: - spec = f.get("measure") or {} - if spec.get("key") not in by_key: - continue - rng = _wn.resolve(spec.get("window"), today) if _wn.normalize(spec.get("window")) else None - if rng is None: - continue # an unresolvable window is unanswerable — never widen to all time - by_window.setdefault((rng[0], rng[1]), []).append(f) - - out = {} - for (dfrom, dto), group in by_window.items(): - keys = tuple(sorted({f["measure"]["key"] for f in group})) - mkey = ("entity", topic, table_key, today, team_id, dfrom, dto, keys) - if mkey not in memo: - try: - memo[mkey] = sem.entity_measure_values(topic, list(keys), date_from=dfrom, - date_to=dto, team_id=team_id, offer=offer) - except Exception as e: # noqa: BLE001 - _ut_measure_err("ut-measure-column", e) - transient = False - try: - from harness import datastore as _ds - transient = not _ds.ready() - except Exception: # noqa: BLE001 - transient = False - if transient: - continue # a warming store must not memoise as a permanent failure - memo[mkey] = None - vals = memo[mkey] - if vals is None: - continue - # C1's empty-window rule for EVERY row, not only the grouped ones. - filled = {pid: sem.entity_zero_fill(vals.get(pid), list(keys), offer) for pid in pids} - for f in group: - k, fkey = f["measure"]["key"], f["key"] - for pid, cell in filled.items(): - if k in cell: - out.setdefault(pid, {})[fkey] = cell[k] - if len(memo) > 100: - memo.clear() - return out - - -def _ut_measure_sets(table_key, views, offer, pids, today, team_id=None): - """`{rule id: [pid, …]}` for the measure CONDITIONS in this database's saved views. - - ⛔ SHIPS WITH THE OFFER, NEVER AFTER IT. `routes_grid` derives `measure_keys` from the same - list, so a non-empty `measures` also opens the measure condition in the filter builder — and a - rule id missing from `measureSets` renders PENDING ("Calculating…") and matches nothing, - permanently. Offer and sets are one feature. - """ - if not offer: - return {} - from harness import semantic as sem - - topic = sem.topic_for_grid(table_key) - admitted = {m["key"] for m in offer} - by_id = {} - for v in views or []: - for rule in sem.entity_measure_leaves((v.get("config") or {}).get("filters"), admitted): - rid = str(rule.get("id") or "") - if rid: - by_id[rid] = rule - if not by_id: - return {} - try: - sets = sem.entity_measure_sets(topic, list(by_id.values()), today, team_id=team_id, - keys_by_id=list(pids), offer=offer) - except Exception as e: # noqa: BLE001 - _ut_measure_err("ut-measure-sets", e) - return {} - return {rid: sorted(int(k) for k in ks if str(k).lstrip("-").isdigit()) - for rid, ks in sets.items()} - - -def ut_label(defn, key, meta=None): - """THE name of a user table, resolved ONCE (wave 20, item 6a). - - `nav_meta`'s rename wins, then the definition's own label, then the key. Every surface that - shows a database name reads through here, because the alternative is what wave 20 found: the - rail showed the renamed name (it reads `nav_meta`) while the automation editor's picker - showed the original (it reads the definition), and neither looked broken. - - ⚠ `set_label` now writes the DEFINITION too, so the two agree at the source. This resolver - stays because it makes every row already stored — renamed before that fix landed — read - correctly today, without a migration. - """ - return ((meta or {}).get(key, {}).get("name") - or (defn or {}).get("label") or key) - - -def nav_meta(session): - """The tenant's nav_meta bucket, read defensively. A store blip must not take a list down.""" - try: - got = session.runtime.get("nav_meta") - return got if isinstance(got, dict) else {} - except Exception: # noqa: BLE001 - return {} - - -@router.get("/tables") -def list_tables(session: Session = Depends(require_session)): - """This session's user tables — the list the '+ New database' surface renders. - - ⭐⭐ W31-T12 (contract C1, D-175's third instance) — ONE DOCUMENT READ, NOT `1 + 2N`. This - route was never ticketed and has the same shape `/nav` and `/automations` were fixed for: - `all_tables` once, then `may_open` per key (another whole-document deep copy each, 28.6 MB on - tenant #0 measured, 703 ms warm) and `records_mutable` per key on top of that — so a tenant - with ten databases paid twenty-one copies to list them. The wall is UNCHANGED and still asked - about every table; it is handed the document this function already holds. See - `user_tables.lend`'s own note for why inlining the predicate is the one fix that is not - available. - """ - ut = _ut() - meta = nav_meta(session) - out = [] - tables = ut.all_tables(st=session.runtime) - lent = ut.lend(session.runtime, **{ut.STORE_KEY: tables}) - for key, t in sorted(tables.items(), - key=lambda kv: (ut_label(kv[1], kv[0], meta) or "").lower()): - if not ut.may_open(key, session.uname, session.admin, st=lent): - continue - out.append({"key": key, "label": ut_label(t, key, meta), - "source": t.get("source") or "Blank", - "recordsMutable": ut.records_mutable(key, st=lent), - "createdBy": t.get("createdBy") or "", - "created": t.get("created") or "", - "fields": [dict(f) for f in (t.get("fields") or [])], - "rowCount": len(t.get("rows") or {})}) - return {"tables": out} - - -#: ⭐⭐ THE ROLLUP SOURCE OFFER — what makes the read-through rollup a FEATURE rather than a -#: capability. Owner 2026-08-09: *"Full field editor: pick topic → metric → window."* -#: -#: ⛔ THE ENGINE SHIPPED WITH NO WRITER. `_clean_rollup` has accepted a `source` bag since -#: 2026-08-09 and `rollup_sql` answers it in one grouped query, but the ONLY thing in the product -#: that ever produced one was a hard-coded field in `odoo_relational.py`. Nothing in the client -#: could make one, so the whole read-through path was reachable by editing Python — the -#: [[artifact-with-no-importer]] shape, twice burned in this repo already. -#: -#: ⚠ DERIVED FROM THE MODEL FILES, NEVER A HAND-WRITTEN LIST. `model/topics/*.yml` and -#: `model/metrics/*.yml` already state which measures belong to which topic and which dims that -#: topic can group by; a second list here would be a second definition of the same fact, and the -#: two would drift the day somebody adds a metric. Same argument `rollup_sql` makes for binding -#: to a metric KEY instead of carrying SQL. -_ROLLUP_CACHE = {} - - -def _rollup_source_offer(): - """`{topics:[…], windows:[…]}` — every (topic, measure, dim) the engine can actually answer. - - ⛔ ONLY COMBINATIONS THAT CAN RESOLVE ARE OFFERED, because a rollup that refuses at COMPUTE - time refuses silently — the cells are simply left blank, hours later, on a column that looks - configured. Two exclusions do real work: - * a topic with NO dims cannot be grouped at all, so it can never key a parent row; - * a CROSS-TOPIC metric (`aov`, `margin_pct`, `returns_pct` — `agg: ratio`/`derived` whose - inputs live elsewhere) is refused by `store_query` the moment a `group_by` is present: - *"cross-topic measures are scalar-only"*. Offering one would mint a column that can only - ever error. - Each dim also declares HOW it keys — by an Odoo id or by its own value — because that is what - the user is matching their own column against, and `payment_state` (a value) and - `partner` (an id) are matched to very different columns. - """ - if _ROLLUP_CACHE.get("offer"): - return _ROLLUP_CACHE["offer"] - from harness import semantic as sem - from harness import windows as W - ut = _ut() - - topics, metrics = sem.topics(), sem.metrics() - by_topic = {} - for key, m in metrics.items(): - # A ratio/derived metric whose parts sit on another topic cannot be grouped — see above. - if m.get("agg") in ("ratio", "derived"): - continue - by_topic.setdefault(m["topic"], []).append( - {"key": key, "label": m.get("label") or key, "format": m.get("format") or "usd", - "description": m.get("description") or ""}) - - out = [] - indeterminate = False - for tkey, t in sorted(topics.items()): - dims = ((t.get("store") or {}).get("dims") or {}) - measures = by_topic.get(tkey) or [] - if not dims or not measures: - continue - # ⛔⛔ W37-T13 / /validate-wave 2026-08-19 — THE SOURCE TABLE MUST BE LIVE, AND THIS DOOR - # DID NOT ASK. The docstring above already promises *"ONLY COMBINATIONS THAT CAN RESOLVE - # ARE OFFERED"*; adding `stock_move` to `datastore.ENTITIES` broke that promise on the one - # door nobody re-read. `semantic._one_binding_offer` — the MEASURE-CATALOG door — refuses - # the same bindings with a stated cause, so the wave shipped **two doors onto one - # vocabulary with only one of them guarded**, and the unguarded one is the picker. - # MEASURED: `_rollup_source_offer()` listed `stock_moves` with both measures and all four - # dims while `royal.duckdb` has no `stock_move` table at all, so choosing it raised - # `CatalogException: Table with name stock_move does not exist!` — five of the fifty - # (topic, measure, dim) triples in `verify_odoo_relational`. A user meets a raw catalog - # error, or a column that looks configured and is blank forever - # ([[permitted-is-not-answerable]]). - # ⚠ REUSES `_source_ready` rather than re-deriving readiness: one question, one - # normalizer. `None` (the store could not be asked) refuses TOO — fail closed, exactly as - # the measure door does. - _tbl = (t.get("store") or {}).get("table") - if _tbl: - _rdy = sem._source_ready(_tbl) - if _rdy is not True: - # ⛔ AND AN INDETERMINATE ANSWER MUST NOT BE CACHED. `_source_ready`'s own comment - # records the measured version of this: a cold container with no mirror yet - # refused every binding, the refusal was memoised, and the grid served no measures - # long after the mirror arrived. Here the same shape would freeze a truncated - # PICKER for the life of the process. - indeterminate = indeterminate or (_rdy is None) - continue - out.append({ - "key": tkey, - "label": t.get("label") or tkey, - "grain": t.get("grain") or "", - "dims": [{"key": dkey, - "label": d.get("label") or dkey, - # `store_query` emits `_id` only when the dim carries a display name - # alongside the key; otherwise the value IS the key. `rollup_sql` handles - # both, and the editor says which so the user matches the right column. - "keyedBy": "id" if d.get("name_col") else "value"} - for dkey, d in dims.items()], - "measures": sorted(measures, key=lambda m: m["label"].lower()), - }) - - # ⚠ THE WINDOW LIST IS `core.user_tables`' OWN, not `harness.windows`'. `ROLLUP_SOURCE_WINDOWS` - # is the validator's closed set and is deliberately NARROWER (it omits the parameterised kinds - # like `last_n_days`, which have nowhere in the bag to carry their `n`). Offering a kind the - # validator refuses would let the editor build a field the save door rejects. - windows = [{"key": k, "label": W.WINDOW_LABELS.get(k, k).format(n="N")} - for k in ut.ROLLUP_SOURCE_WINDOWS] - offer = {"topics": out, "windows": windows} - # ⚠ Cache only a DEFINITE answer (see the readiness block above): if any topic was dropped - # because the store could not be asked, this offer is a snapshot of an unknown, not a fact. - if not indeterminate: - _ROLLUP_CACHE["offer"] = offer - return offer - - -@router.get("/tables/rollup-sources") -def rollup_sources(session: Session = Depends(require_session)): - """The topic → metric → dim → window offer the rollup field editor renders. - - ⚠ DECLARED ABOVE EVERY `/tables/{table_key}/…` ROUTE, and kept there deliberately. FastAPI - matches in declaration order, so the day somebody adds a bare `GET /tables/{table_key}` below - this line it still resolves; added ABOVE it, this endpoint would silently start arriving as - `table_key='rollup-sources'` and 404 from the table wall. There is no such route today — - this is the cheap ordering that keeps it from mattering. - - ⛔ TENANT-SCOPED, AND IT WAS NOT WHEN FIRST WRITTEN. `sem.topics()` reads the GLOBAL model - files, so nurilab and gtmlab were served the full Odoo offer — they would have seen "Live - Odoo data" in the field editor and been able to build a column that can only ever be blank, - because there is no mirror behind it. Three comments (here, in `apiBridge` and on the - `rollupSourceOffer` prop) each asserted that a tenant with nothing connected receives `[]`, - and the mode switch's "render only when there is a choice" guard is built on that promise. - ⚠ `odoo_relational.is_royal` is the authority, reused rather than re-decided: it is already - what `refresh` consults to decide whether these tables may exist at all, and a second copy of - the rule would be a second answer the day a tenant gains a mirror. - """ - import odoo_relational - if not odoo_relational.is_royal(session.tenant): - return {"topics": [], "windows": []} - return _rollup_source_offer() - - -@router.post("/tables", status_code=201) -def create_table(body: dict = Body(default=None), - session: Session = Depends(require_session)): - ut = _ut() - body = body or {} - label = str(body.get("label") or "").strip() - if not label: - raise err(400, "bad_label", "give the database a name") - if not session.runtime.available(): - raise err(503, "store_unavailable", - "the tenant store is unavailable. Nothing was created") - source = body.get("source") - try: - key = ut.create(label, session.uname, fields=body.get("fields"), - source=source, st=session.runtime) - except Exception: - raise err(503, "store_unavailable", - "the tenant store refused the write. Nothing was created") - if not key: - raise err(400, "refused", - f"could not create it. The name may be empty or this tenant already has " - f"{ut.MAX_TABLES} databases") - return {"key": key} - - -@router.patch("/tables/{table_key}") -def patch_table(table_key: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Rename a database — IN ITS DEFINITION (wave 20, item 6a). - - ⚠ THE RENAME DOOR IN THE NAV WRITES `nav_meta` AND MUST ALSO CALL THIS. `nav_meta` is the - nav's display layer; the definition is what the automation editor's database picker, the - schema drawer and every future reader see. A rename that lands in only one of them leaves a - picker that is confidently wrong rather than obviously stale. Posted to the wave doc as an - amendment for whoever owns that door. - """ - defn = _defn_or_refuse(session, table_key) - ut = _ut() - if not (session.admin or defn.get("createdBy") == session.uname): - raise err(403, "forbidden", "only the database's creator or an admin can rename it") - label = ut.set_label(table_key, (body or {}).get("label"), st=session.runtime) - if not label: - raise err(400, "bad_label", "give the database a name") - return {"key": table_key, "label": label} - - -@router.get("/tables/{table_key}/footprint") -def table_footprint(table_key: str, session: Session = Depends(require_session)): - """What dies with this database — the confirm dialog's disclosure (wave 21, item 6a / C3). - - Counts drill to the SAME buckets `user_tables.delete` cleans; a dialog listing categories - without numbers would break [[no-unverifiable-aggregates]] at the scariest moment. Walled - like the delete itself: only someone who could delete may case the joint. - - ⭐ W33-T01 / D-213: the wall and `createdBy`/`fields` come off the PROJECTION; only the row - COUNT needs the whole document, and only on a materialised table.""" - defn = _defn_or_refuse(session, table_key, defs_only=True) - if not (session.admin or defn.get("createdBy") == session.uname): - raise err(403, "forbidden", "only the database's creator or an admin can delete it") - s = session.runtime - views, fields = set(), len(defn.get("fields") or []) - try: - bucket = s.get(f"{table_key}_table_workspace") or {} - for _u, ws in bucket.items(): - if isinstance(ws, dict): - views |= set((ws.get("views") or {}).keys()) - fields += len(ws.get("fields") or {}) # per-user custom/measure strata - except Exception: - pass - import core.shares as shares - g = shares.grants("database", table_key, st=s) - auto = [] - try: - import automation_engine as engine - for aid, d in (engine.all_definitions(s) or {}).items(): - if (d.get("config") or {}).get("targetTable") == str(table_key): - auto.append({"id": str(aid), "name": d.get("name") or str(aid)}) - except Exception: - pass - # ⛔ THE ROW COUNT IS THE ONE FIELD THAT NEEDS THE WHOLE DOCUMENT, and it needs it only where - # the rows are actually stored here. A read-through database keeps `rows: {}` by construction, - # so `len(...)` answered **0** for it before this change and answers 0 now — identical, and the - # projection is not what makes it wrong. - # ⚠ 0 IS A WRONG NUMBER FOR A READ-THROUGH GRID and always was (`ut_odoo_gl_lines` would say 0 - # in a dialog headed "what dies with this database"). Booked rather than fixed here: this - # ticket is a read-path change and correcting it means asking the mirror for a `count(*)` - # inside a confirm dialog. See the `PENDING:` line in `mailbox/A.md`. - if _ut().materialises(table_key, st=s, defn=defn): - rows = len((_ut().get(table_key, st=s) or {}).get("rows") or {}) - else: - rows = 0 - return {"rows": rows, "fields": fields, "views": len(views), - "sharedUsers": len(g.get("entries") or []), - "automations": sorted(auto, key=lambda a: a["name"].lower())} - - -@router.delete("/tables/{table_key}") -def delete_table(table_key: str, session: Session = Depends(require_session)): - """CREATOR OR ADMIN — checked explicitly (wave 21, item 6a / C3). - - ⛔ The wave-20 docstring said "the same actors may_open admits" and that stopped being the - creator-or-admin set the day de5037f taught `may_open` to admit share GRANTEES: a view-role - grantee could reach this route and delete the database somebody shared with them. The wall - is now the definition's own `createdBy`, the same check the rename route always had. - - Deletion cleans the artifact families server-side (`user_tables.delete` lists them) and - DISABLES bound automations with a status note — never deletes them. The client's confirm - dialog disclosed `/footprint` first; the server cannot tell a click from a plan, so the - dialog is a product requirement, not a formality.""" - defn = _defn_or_refuse(session, table_key) - if not (session.admin or defn.get("createdBy") == session.uname): - raise err(403, "forbidden", "only the database's creator or an admin can delete it") - try: - import automation_engine as engine - engine.disable_for_table(session.runtime, table_key) - except Exception: - pass - try: - _ut().delete(table_key, st=session.runtime) - except Exception: - raise err(503, "store_unavailable", "the delete did not land. Try again") - return {"ok": True} - - -#: ⭐ 2026-08-07 — tenants whose Instagram tables THIS PROCESS has already brought forward. -_IG_FORWARDED = set() - - -def _ig_forward(session): - """Bring this tenant's Instagram tables onto the current schema, at most once per process. - - ⛔ WHY A MIGRATION RUNS ON A READ AT ALL, when the module's own rule is that it rides the WRITE - path. `ut_ensure` calling it is right for a schema an automation is about to append to, and - useless for a change a PERSON is waiting to see: the owner's report was *"the first field is - still blank"*, and "re-save the automation and it will fix itself" is not an answer to that. - The write path stays exactly as it was — this is a second door to the same idempotent call, - not a replacement for it. - - ⚠ BOUNDED THREE WAYS, because a write on a read is otherwise how a grid gets slow: once per - tenant per process; `migrate_ig_tables` returns without a write when every table is already - current (the common case after the first read); and a failure is SWALLOWED — a migration must - never be the reason a database will not open. - - ⚠ THE TENANT IS MARKED BEFORE THE ATTEMPT, deliberately. A migration that raises must not be - retried on every subsequent read of every table for the life of the process — the write path is - still the backstop, so the cost of skipping is a delay, while the cost of retrying is a failing - store call on the hot path of a grid that is trying to render. - """ - tenant = str(getattr(session, "tenant", "") or "") - if tenant in _IG_FORWARDED: - return - _IG_FORWARDED.add(tenant) - try: - import automation_engine as engine - engine.migrate_ig_tables(session.runtime, log=lambda *_a: None) - except Exception as e: # noqa: BLE001 - print(f"[tables] ig forward-migration skipped: {type(e).__name__}: {e}") - - - -#: Above this many characters a `json` cell is replaced by a stand-in in the LIST envelope. Sized -#: so an ordinary config document (a few hundred bytes) is untouched while a vendor response is -#: not — the shape this exists for is one already-paid provider payload per row. -JSON_LIST_MAX = 400 - - -def _thin_json(fields, merged, table_key=""): - """Replace oversized `json` cells with a stand-in for the LIST response. Pure; returns a copy. - - ⚠ THE STAND-IN IS ITSELF VALID JSON and carries the byte count, so the grid preview reads - `{...} 3 keys` rather than a broken brace, and a reader can see the column holds something - large rather than something empty. `_truncated` is what the viewer keys its fetch on. - - ⭐ THE STAND-IN CARRIES ITS OWN `_url`, which is what keeps this change small AND correct: the - viewer needs no table key, no record id and no new props threaded down through three - components to find the document — the route that removed the value says where it went. One - writer of that address instead of a server rule and a client rule that must agree forever. - """ - json_keys = [str(f.get("key")) for f in (fields or []) - if str(f.get("type") or "") == "json"] - if not json_keys: - return merged - out = {} - for pid, cells in (merged or {}).items(): - row = cells - for key in json_keys: - raw = cells.get(key) - if isinstance(raw, str) and len(raw) > JSON_LIST_MAX: - if row is cells: - row = dict(cells) - row[key] = json.dumps({ - "_truncated": True, "bytes": len(raw), - "_url": f"/api/v1/tables/{table_key}/rows/{pid}/fields/{key}"}) - out[pid] = row - return out - - -@router.get("/tables/{table_key}/rows") -def table_rows(table_key: str, session: Session = Depends(require_session)): - """The `/customers`-shaped envelope for one user table: `{fields, rows, today, pulled_at, - identity}` — so the client's generic topic fetch consumes it with zero new parsing. - - ⚠ THE MERGE ORDER IS THE CONTRACT. `rows_from_pool` sources an overlay-typed field's cell - from the OVERLAY stratum only (that is what makes a custom column render standalone) — a - user table's base values live in its DEFINITION rows, so they are layered UNDER the user's - overlay edits here: base first, overlay wins. Without this every base cell reads empty - (found by this route's own gate check, not by luck).""" - import aios_grid - - _ig_forward(session) - # ⭐⭐ W33-T03 / D-214 — ONE READ OF THE TENANT DOCUMENT FOR THE WHOLE REQUEST, MEASURED. - # This route asked for it FOUR times: the wall (via `scoped_pool`), then `limit_report` TWICE - # (`row_limit` calls `materialises` and then `is_connected`, and each takes its own whole copy), - # then `records_mutable` at the envelope. Each is ~703 ms warm on tenant #0, and all four ask - # about the SAME document in the SAME request. The lend is the fix W31 QA already built for - # `_defn_or_refuse`; this threads it through the three sites that never got it. - # ⚠ SAFE HERE FOR THE SAME REASON IT IS SAFE THERE: this is a pure READ route. A lend is a - # PRE-WRITE snapshot, and handing one to a path that writes and then reads back would report the - # value it replaced (contract C5). - _lent = _ut().lend(session.runtime) - g = ut_assembly(session, table_key, st=_lent) - # ⛔ THE OVERLAY WAS NEVER ACTUALLY MERGED, and the docstring above has described the merge - # this line does not perform since the route was written (owner item 3, 2026-08-09: - # *"Using the swipe, fast doesn't register the CHANGE. I went back and it all got reseted"*). - # - # `merged` was built from `rows_src` ALONE — the DEFINITION rows. But `rows_from_pool` - # sources an overlay-typed field's cell from this dict, and a CUSTOM column on a `ut_*` table - # is overlay-typed by construction, so it looked up a key that could not be there and every - # such cell rendered blank. - # - # ⭐ THE WRITES WERE NEVER LOST — MEASURED. `ws['overlays']` holds - # `{"1": {"custom_geography_yf6vi": "Jakarta"}, ...}` for ten rows: the owner's swipes landed - # in the store exactly as they should. Only the READ-BACK dropped them, which is why the - # value survived the gesture, vanished on reload, and looked like "it reset itself" — and - # why `patch_row`'s `_took()` then reported a perfectly good write as `refused`. - # - # ⚠ OVERLAY WINS, base underneath — the order the docstring already specifies. A definition - # value must not shadow an edit the user has made on top of it. - # ⚠ AND IT IS THIS USER'S OWN OVERLAY (`table_workspace` is keyed by `ctx.uname`), so this - # widens what a caller can SEE by exactly their own edits and nothing else. - _overlays = (g.get("ws") or {}).get("overlays") or {} - merged = {} - for _r in g["rows_src"]: - _pid = str(_r["pid"]) - _cells = {k: v for k, v in _r.items() if k != "pid"} - _ov = _overlays.get(_pid) - if isinstance(_ov, dict): - _cells.update(_ov) - merged[_pid] = _cells - # ⭐⭐ W38-T16 — THE TENANT-WIDE STRATUM GOES ON TOP, and the order is the contract rather - # than a preference. A shared column is one the whole permitted audience must read the SAME - # value in — that is the entire reason `shared_overlay` exists — so a stale per-user value - # left under the same key must not win. `routes_odoo_tables` layers the two in this exact - # order (`overlays.setdefault(pid, {}).update(cells)`) and the two doors must agree. - # ⛔ NEVER `setdefault` ON `merged`: these cells are already narrowed to this session's pids, - # but inventing a row id here would put a row in the payload that `rows_src` never admitted. - for _pid, _cells in (g.get("shared_cells") or {}).items(): - if str(_pid) in merged and isinstance(_cells, dict): - merged[str(_pid)].update(_cells) - # ⭐⭐ 2026-08-10 (owner: *"wth is going on, why does it take forever to load now?"*) — THE - # JSON DOCUMENTS DO NOT RIDE THE LIST. - # - # MEASURED on nurilab, and the numbers are the whole argument: `source_payload` is **95.6% - # to 98.5%** of every IG grid's bytes — `ut_ig_post_snapshots` shipped **11.6 MB of a 12.2 MB - # response**, `ut_ig_snapshots` 1.85 MB of 2.03 MB, the profile table 1.83 MB of 2.11 MB. The - # grid renders those cells as a SIXTY-CHARACTER preview (`display.jsonPreview`), so the whole - # vendor response crossed the wire, was parsed by the browser and held in memory purely so a - # clipped first line could be drawn. - # - # ⚠ IT IS NOT A CAP AND NOTHING IS LOST. The full document is served by - # `GET /tables/{key}/rows/{pid}/fields/{fkey}`, which the JSON viewer fetches when it opens — - # the one place a person actually reads it, for the one row they opened. The cell that rides - # the list is a VALID small document saying what it stands for, so the preview renders - # honestly instead of showing half a truncated brace. - # ⛔ ONLY `json` COLUMNS, and only over the threshold: a small document still travels whole, - # so a tenant using `json` for a short config sees no change at all. - rows = aios_grid.rows_from_pool( - g["rows_src"], g["fields"], _thin_json(g["fields"], merged, table_key), derived=g["derived"]) - # ⭐⭐ WAVE-34 (R13) — the per-cell enrichment STATE rides the row, beside `_created`/`lat`/ - # `lon`. See `_stamp_ai_states` for why it is a row key rather than a map beside `rows`. - _stamp_ai_states(table_key, g["fields"], rows, st=_lent) - # ⭐ R6's SECOND SENTENCE, ON THE WIRE (W30-T29). *"if there is lag or it can't be done, you - # need to explicitly tell me why and recommend a fix."* A ceiling that still applies to this - # database says so here, with its cause and the recommendation, rather than waiting to be - # discovered as a refused paste. `None` for a connected source, and an EMPTY LIST is the - # honest answer for a table nothing limits — never an absent key, which a client cannot tell - # apart from an older server. - _report = _ut().limit_report(table_key, st=_lent) - # ⭐ C4 / D-138 — THE DOCUMENTS PRODUCER. Absent since EXIT-6 deleted `app.py`, which was the - # only thing that ever set this key; the write door never stopped working and every client - # half is complete, but all six `onDoc*` handlers read `payload?.docs ? … : undefined`, so a - # missing key has been silently switching the feature off. ⛔ ONE shared serialiser with the - # customer scope — `grid_events.docs_for` — never a twin here. - from core import grid_events as _ge - return {"fields": g["fields"], "rows": rows, "today": g["today"], - "docs": _ge.docs_for(g["pids"], scope_key=table_key, uname=session.uname, - admin=session.admin, st=session.runtime), - "pulled_at": time.strftime("%Y-%m-%d %H:%M"), - "identity": {"pid": "pid"}, - "scope": {"table": table_key, "rowCount": len(rows)}, - "limits": [_report] if _report else [], - "recordsMutable": _ut().records_mutable(table_key, st=_lent)} - - -#: The per-cell provenance a row carries on the wire, one key per enrichment column. -#: ⛔ A ROW KEY RATHER THAN A SIBLING MAP, and the choice is load-bearing rather than cosmetic. -#: A `{colId: {pid: state}}` map beside `rows` would need a new PROP on `RecordDetail` and a new -#: argument at `CustomerGrid`'s call site, both in another lane's fence, to reach the two surfaces -#: that must paint it. The row already carries `_created`, `lat` and `lon` for exactly this -#: reason, so every reader already tolerates keys that are not columns, and both surfaces hold the -#: row already. ⚠ COLLISION-PROOF BY CONSTRUCTION: `_clean_field` strips leading underscores off -#: every field key, so no column can ever be called `_ai_*`. -AI_STATE_PREFIX = "_ai_" - - -def _stamp_ai_states(table_key, fields, rows, st=None): - """Add `_ai_` to each row for every `ai_enrich` column. Mutates and returns `rows`. - - ⛔ A PROJECTION, NOT THE MARK SET. The stratum holds a hash, a model, a timestamp, a token - count and an error per cell; a browser needs ONE WORD to paint a state, and shipping the rest - would grow this payload by a dict per enriched cell for data no reader reads. The vocabulary - is `agent`/`human`/`stale`/`error` (`api/ai_enrich.py::cell_state`), which is also what the - RUNNER obeys, so the badge and the behaviour cannot disagree about whose cell it is. - - ⚠ AN ABSENT KEY MEANS `empty`, and only non-empty states are stamped: a table with no - enrichment column is untouched, and a freshly created column adds nothing until something - runs. ⛔ `stale` is DERIVED here rather than stored, so it is computed against TODAY'S row - instead of against whatever was true when the value was written. - """ - cols = [f for f in (fields or []) if str(f.get("type") or "") == "ai_enrich"] - if not cols: - return rows - import ai_enrich as _ae - for field in cols: - col = str(field.get("key") or "") - marks = _ut().ai_enrich_marks(table_key, col, st=st) - cfg = field.get("aiEnrich") if isinstance(field.get("aiEnrich"), dict) else {} - for row in (rows or []): - if not isinstance(row, dict): - continue - state = _ae.cell_state(marks.get(str(row.get("pid"))), cfg, row, col) - if state != "empty": - row[AI_STATE_PREFIX + col] = state - return rows - - -@router.get("/tables/{table_key}/rows/{pid}/fields/{fkey}") -def table_cell(table_key: str, pid: str, fkey: str, - session: Session = Depends(require_session)): - """ONE cell, whole — the other half of `_thin_json`. - - ⛔ WITHOUT THIS THE THINNING WOULD BE A CAP, and a cap on data somebody already paid a vendor - for is exactly what this product refuses everywhere else. The list ships a stand-in; the JSON - viewer opens this for the one row a person is actually reading. - - ⚠ SAME PERMISSION WALL AS THE LIST, reached the same way (`ut_assembly` resolves the session's - view of the table), so this cannot become a side door onto a table the caller may not open — - which is the failure a "just fetch the raw cell" helper invites. - ⚠ THE OVERLAY WINS, exactly as it does in `table_rows`: a user who has typed over a cell must - read back what they typed, not the definition value underneath it. - """ - g = ut_assembly(session, table_key) - field = next((f for f in (g.get("fields") or []) if str(f.get("key")) == str(fkey)), None) - if field is None: - raise err(404, "unknown field", f"{fkey!r} is not a column on this database") - row = next((r for r in g["rows_src"] if str(r.get("pid")) == str(pid)), None) - if row is None: - raise err(404, "unknown record", f"no record {pid!r} in this database") - overlay = ((g.get("ws") or {}).get("overlays") or {}).get(str(pid)) or {} - value = overlay.get(fkey, row.get(fkey)) - return {"table": table_key, "pid": str(pid), "field": str(fkey), - "value": "" if value is None else str(value)} - - -@router.post("/tables/{table_key}/rows", status_code=201) -def add_row(table_key: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Append a row — or RESTORE one under its old id (contract C-ADDROW / C-UNDO). - - ⚠ THE ANSWER IS ALWAYS THE ID THAT WAS STORED, never the one that was asked for. An undo - that requested `rid: 7` and got 12 because 7 had been re-used must find that out from the - response rather than assume; the client re-anchors on what came back. - """ - _records_or_refuse(session, table_key) - ut = _ut() - values = (body or {}).get("values") or {} - if not isinstance(values, dict): - raise err(400, "bad_values", "values must be an object of {fieldKey: value}") - try: - rid = ut.add_row(table_key, values, session.uname, st=session.runtime, - rid=(body or {}).get("rid")) - except Exception: - raise err(503, "store_unavailable", "the row was not saved. The store refused") - if rid is None: - # C3 (wave 25): `add_row` also refuses a profile cell that is not a handle, so the cap - # sentence alone would misdirect — the reader would go and count rows. Ask the same - # validator the law used rather than re-deciding here (one rule, two voices). - pf = ut.profile_field(table_key, st=session.runtime) - if pf and pf["key"] in values: - _h, ok = ut.normalize_profile(values[pf["key"]], pf["profile"].get("source")) - if not ok: - raise err(400, "refused", - f"{str(values[pf['key']])[:80]!r} is not an Instagram profile. " - f"{pf.get('label') or pf['key']!r} takes a handle (@name) or a " - f"profile link (instagram.com/name)") - raise err(400, "refused", - f"row refused. The table may be at its {ut.MAX_ROWS}-row cap") - _refresh_relations(session) - return {"rid": rid, "pid": int(rid)} - - -@router.post("/tables/{table_key}/rows/import", status_code=201) -def import_rows(table_key: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """⭐⭐ WAVE-29 T25 (owner item 6) — the IMPORT door: N mapped rows, ONE store write. - - Body: `{"rows": [{fieldKey: value, ...}, ...]}` — already MAPPED by the client's dialog, so a - spreadsheet column name never reaches the store. APPEND-ONLY in v1: every row is a new record, - nothing is matched or overwritten, and the dialog says so before the button is pressed. - - ⛔ COMPUTED COLUMNS ARE REFUSED HERE, NOT FILTERED. `is_computed_cell` is the same predicate - the cell wall uses (one evaluator for one question), and a rollup or formula key arriving in - an import is not a stray to be tidied away — it means the client offered a target it should - not have, and silently dropping it would leave the user looking for a column of values that - never arrived. The refusal names the column. - - ⛔ ATOMIC. `add_rows` writes nothing unless the whole batch fits under `MAX_ROWS` and every - profile cell validates, because a half-imported file is the worst outcome available: the user - cannot tell which rows landed without reconciling the spreadsheet by hand. - """ - _records_or_refuse(session, table_key) - ut = _ut() - rows_in = (body or {}).get("rows") - if not isinstance(rows_in, list) or not rows_in: - raise err(400, "bad_rows", "rows must be a non-empty array of {fieldKey: value} objects") - if any(not isinstance(r, dict) for r in rows_in): - raise err(400, "bad_rows", "every row must be an object of {fieldKey: value}") - defn = _defn_or_refuse(session, table_key) - by_key = {f["key"]: f for f in (defn.get("fields") or [])} - asked = {k for r in rows_in for k in r} - unknown = sorted(k for k in asked if k not in by_key) - if unknown: - raise err(400, "unknown_field", - f"this database has no column {unknown[0]!r}") - computed = sorted(k for k in asked if ut.is_computed_cell(by_key[k])) - if computed: - label = by_key[computed[0]].get("label") or computed[0] - raise err(400, "computed_field", - f"{label!r} is worked out from other columns, so it cannot be imported into") - # ⛔ W29-T81 — THE TYPE WALL, AND IT LIVES HERE BECAUSE THE ONLY OTHER ONE IS IN THE BROWSER. - # `coerceClipboardValue` refuses "seventeen-ish" at an `int` column in the dialog; curl, a - # second client, or a future importer met no wall at all and the string landed verbatim in a - # typed column, where the grid then painted it as a fabricated `0`. Refused whole and BEFORE - # `add_rows`, matching this door's own atomicity: a half-imported file is the outcome every - # rule on this route exists to prevent. The sentence names the row and the column, because - # "invalid value" sends somebody hunting through 2,000 lines of spreadsheet. - for index, row in enumerate(rows_in): - for key, value in row.items(): - why = ut.cell_type_refusal(by_key[key], value) - if why: - raise err(400, "bad_value", f"row {index + 1}: {why}. Nothing was imported") - try: - made = ut.add_rows(table_key, rows_in, session.uname, st=session.runtime) - except Exception: - raise err(503, "store_unavailable", "nothing was imported. The store refused") - if made is None: - raise err(400, "refused", - f"nothing was imported. {len(rows_in)} rows would take this database past " - f"its {ut.MAX_ROWS}-row cap, or a profile column rejected a value") - _refresh_relations(session) - return {"imported": len(made), "pids": [int(r) for r in made]} - - -# --------------------------------------------------------------------------------------------- -# THE SHARED FIELD SCHEMA (contract C-FIELD, owner ruling R2) -# --------------------------------------------------------------------------------------------- -# R2: a `ut_*` table's fields are the TABLE'S schema — everyone with access sees the same -# columns, the creator or an admin edits them, and a per-field `editRole` can open ONE column's -# definition to everyone without handing over the table. This supersedes wave 17's "fields are -# per-user" law for this path only; the connector scopes keep their own model. -# -# ⚠ WHY IT MATTERS BEYOND TIDINESS: a grid add-field on a `ut_` scope used to land in the -# per-user workspace overlay, which is why the automation editor's "Automation column" picker -# could not see a column the user had just created — it reads the DEFINITION. Same defect shape -# as the rename (item 6a): two places to look, and the surfaces disagreed silently. - -def _field_or_refuse(session, table_key, fkey=""): - """The schema wall. `_defn_or_refuse` first (404/403 on the table), then the per-field rule. - - ⭐ W33-T01 / D-213: definitions only. Everything read off `defn` here is `fields` and - `createdBy`; the returned value is DISCARDED by all three callers (they re-fetch what they - write through the `user_tables` write doors), so no projected snapshot survives into a write. - ⚠ It is a SCHEMA wall on a write route, not a row-write wall — the distinction contract C5 - draws is about the snapshot reaching a read-BACK, and this one does not escape the function.""" - defn = _defn_or_refuse(session, table_key, defs_only=True, scope_applied=True) - ut = _ut() - if not ut.is_user_table(table_key, st=session.runtime): - raise err(400, "not_a_user_table", - "only a user-created database has an editable schema. A connected source " - "owns its own columns") - # ⭐⭐ W36-T21 — A HIDDEN COLUMN IS NOT EDITABLE, AND THIS IS THE DOOR THAT HAD TO SAY SO. - # `EventCtx.hidden_keys` walls the events transport; the REST schema routes (rename, retype, - # delete a column) do not pass through it at all. Without this an account that could not SEE - # `unit_cost` could still DELETE it for the whole tenant — the loudest possible version of a - # wall that exists on one wire only. ⚠ Read the closure off the DECLARED fields, which is what - # `routes_admin` validates a stored `hiddenFields` list against. - if fkey and str(fkey) in _ut_hidden(session, table_key, defn.get("fields") or []): - raise err(403, "forbidden", - "this database has no column by that name that you may edit") - if fkey and not ut.may_edit_field(table_key, fkey, session.uname, session.admin, - st=session.runtime): - field = next((f for f in (defn.get("fields") or []) - if f.get("key") == str(fkey)), None) - if isinstance((field or {}).get("automation"), dict) \ - and field["automation"].get("preset") is True: - # ⚠ THIS BRANCH NARRATES `may_edit_field`, it does not re-decide (the `and not - # ut.may_edit_field` above is the wall). Said out loud because the sentence itself - # went stale on 2026-08-09: preset ROLLUPS became editable by owner ruling, so a - # blanket "pre-set fields are locked" would now be the server explaining a refusal - # it did not make — `preset_editable` is the one predicate that answers this. - raise err(403, "preset_field_locked", - "this is a pre-set column, so its name and type are fixed; you may sort, " - "filter or hide it, edit any Rollup column, and add your own columns") - raise err(403, "forbidden", "that column can only be changed by the database's creator " - "or an admin") - if not fkey and not (session.admin or defn.get("createdBy") == session.uname): - raise err(403, "forbidden", "only the database's creator or an admin can add a column") - return defn - - -@router.post("/tables/{table_key}/fields", status_code=201) -def add_field(table_key: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - _field_or_refuse(session, table_key) - ut = _ut() - field = ut.add_field(table_key, body or {}, st=session.runtime) - if not field: - # ⭐ D-46 CLOSED (wave 23) — the C8 flow law gets its OWN sentence. `add_field` answers - # None for every refusal, so this route said "check the name and type" to somebody whose - # name and type were fine and whose automation column named a flow that does not exist. - # A refusal that misdirects is worse than a bare 400: it sends the reader to look at the - # one thing that was never wrong. Checked HERE, in the route's own words, because the - # law itself stays enforced in `user_tables.flow_bound` — this narrates it, never - # re-implements it (a second copy of the rule is how two doors start disagreeing). - raise err(400, "refused", - _refusal_sentence(ut, session, body or {}, table_key=table_key)) - if field.get("type") == "link": - synced = ut.sync_reciprocal_link(table_key, field["key"], st=session.runtime) - field = synced.get("field") or field - # ⭐⭐ 2026-08-09 — `rollup` REFRESHES TOO, and the omission was invisible until this route - # became reachable for one. It was gated on `link` alone, while `patch_field` and - # `delete_field` next door refresh unconditionally — so a newly created Rollup got its first - # fold from `_store_resync_loop`, which sleeps 1800 s BEFORE its first pass (D-107's shape). - # The user would have created the column, watched a 201 come back, and read a blank cell for - # half an hour: *"the Rollup doesn't work"*, arriving through the door opened to fix it. - # ⚠ Still conditional rather than unconditional: a pass deep-copies every table and row in - # the tenant, and adding a text column has nothing to fold. The condition is now "is this - # field relational", which is the question that was always meant. - if field.get("type") in ("link", "rollup"): - _refresh_relations(session) - return _with_dropped(ut, {"field": field}, body) - - -def _with_dropped(ut, out, body): - """Attach the NAMED list of config keys the validator did not keep (wave 34, R13 / W34-T51). - - ⛔⛔ THIS EXISTS BECAUSE THE VALIDATOR HAS NO ERROR CHANNEL AND CANNOT GROW ONE. Every bag - cleaner in `core/user_tables.py` returns `dict | None` and drops unknown keys in silence, and - `verify_fields_contract` asserts that they do -- so the drop is correct and the SILENCE is the - defect. T51's contract is that an unknown config key is dropped **and named**, so the naming - rides the response beside the accepted field rather than inside the validator. - - ⚠ OMITTED WHEN EMPTY, deliberately: an always-present `dropped: []` teaches every reader to - ignore the key, which is how a report stops being read before it stops being true. - """ - dropped = ut.ai_enrich_dropped_keys((body or {}).get("aiEnrich")) - if dropped: - out = dict(out) - out["dropped"] = dropped - return out - - -def _refusal_sentence(ut, session, body, table_key="", fkey=""): - """Why was this column refused? The specific reason when we can name one, the general list - otherwise — never a specific-sounding guess.""" - # ⭐ WAVE-34 (R13): the enrichment column's own sentence, named BEFORE the automation bag - # below. `_clean_field` DERIVES `field.automation` for this kind, so a refused enrichment - # column would otherwise be explained by the flow law -- "pick a flow, or make this an - # ordinary column" -- which is the D-46 misdirection exactly, pointing at a control the user - # never touched. - if str((body or {}).get("type") or "").strip().lower() == "ai_enrich": - bag = (body or {}).get("aiEnrich") - if not isinstance(bag, dict) or not str(bag.get("prompt") or "").strip(): - return ("an AI enrichment column needs a prompt. It is the only thing that can " - "produce a value here, so a column without one would stay empty forever") - # ⭐ C3 (wave 25, R7): the one-profile-per-table refusal NAMES THE EXISTING COLUMN, which is - # what the contract asks for and what makes it actionable — "at most one" sends the reader - # hunting through a 40-column schema for a flag they cannot see from the header. - if isinstance((body or {}).get("profile"), dict): - # ⚠ THE TYPE IS NAMED FIRST, and the order is the point. Both refusals can be true at - # once (an `int` profile column on a table that already has a profile column), and the - # TYPE is the one that is wrong about what the caller just sent — unconditionally, no - # matter what else is on the table. Answering "you already have one" to somebody whose - # real mistake was the column type sends them to fix the wrong thing, which is the - # misdirection D-46 closed one door over. - if str((body or {}).get("type") or "text").strip().lower() != "text": - return ("a profile column is a flag on an ordinary TEXT column. It validates what " - "is typed into it, which it can only do for text") - existing = ut.profile_field(table_key, st=session.runtime) if table_key else None - if existing and existing.get("key") != str(fkey): - return (f"this database already has a profile column: " - f"{existing.get('label') or existing.get('key')!r}. A database has at most " - f"one, so the automation knows which handle to enrich; edit that column, or " - f"take the flag off it first") - bag = (body or {}).get("automation") - if isinstance(bag, dict): - flow = str(bag.get("flowId") or "").strip() - if not flow: - return ("an automation column has to name the automation that fills it. Pick a " - "flow, or make this an ordinary column") - if not ut.flow_bound(bag, st=session.runtime): - return (f"this column names automation {flow!r}, which does not exist in this " - f"workspace. It may have been deleted; pick a flow that is still there") - kind = str((body or {}).get("type") or "").strip() - if kind and kind not in ut.UT_FIELD_TYPES: - return (f"{kind!r} is not a column type here (types: " - f"{', '.join(sorted(ut.UT_FIELD_TYPES))})") - return (f"the column was refused. Check the name and type, or the table may be at its " - f"{ut.MAX_FIELDS}-column cap (types: {', '.join(sorted(ut.UT_FIELD_TYPES))})") - - -@router.patch("/tables/{table_key}/fields/{fkey}") -def patch_field(table_key: str, fkey: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Edit one column's definition, and MIGRATE its values when options are renamed. - - ⛔ A CHOICE RENAME IS AN EXPLICIT MAPPING, NEVER A DIFF (contract C-RENAME). `{renames: - [{from, to}]}` arrives alongside the new options list, because a diff cannot tell "renamed - Blue to Navy" from "deleted Blue, added Navy" — and guessing wrong empties the column and - every saved view that filtered on it. - """ - _field_or_refuse(session, table_key, fkey) - ut = _ut() - body = body or {} - migrated = None - renames = body.get("renames") - if renames: - try: - migrated = ut.rename_choice_values(table_key, fkey, renames, st=session.runtime) - except Exception: - raise err(503, "store_unavailable", "the rename did not land. Try again") - # The per-user workspace strata and any view filter naming the old value are the OTHER - # half of C-RENAME and belong to `core.table_store`. Called only if it is there: an - # enumerator's mirror waits for its counterpart rather than guessing at its shape, and a - # missing counterpart must not lose the half that DID land. - try: - import core.table_store as table_store - fn = getattr(table_store, "rename_choice_values", None) - if callable(fn): - fn(table_key, fkey, renames, st=session.runtime) - migrated = dict(migrated or {}, workspace=True) - except Exception: # noqa: BLE001 - migrated = dict(migrated or {}, workspace=False) - field = ut.patch_field(table_key, fkey, body, st=session.runtime) - if not field: - # C3: the same narrator as `add_field`. Flagging a SECOND column is the same act as - # adding one, so it must get the same sentence naming the column that already holds the - # flag — a patch that answered "check the name and type" would send the reader to the - # one thing that was never wrong (the D-46 lesson, one door over). - raise err(400, "refused", - _refusal_sentence(ut, session, body, table_key=table_key, fkey=fkey)) - synced = ut.sync_reciprocal_link(table_key, fkey, st=session.runtime) - field = synced.get("field") or field - _refresh_relations(session) - out = {"field": field} - if migrated is not None: - out["migrated"] = migrated - return _with_dropped(ut, out, body) - - -def _fire_on_change(table_key, pid, changed, session): - """Run any `on_change` enrichment column whose prompt names a cell that just moved. - - ⛔ ONE DEFINITION READ FOR THE WHOLE WRITE, and that is the point rather than an optimisation. - `automation_engine.grid_hook` calls `all_definitions(st)` once PER EVENT, which turns a - 20,000-row import into 20,000 whole-document reads on the single process this product runs - (`D-134`, and `W34-T54`'s own `how:` says not to rebuild it). `on_change_fields` is pure and - takes the definition, so this reads once and asks about every column. - - ⛔ AND IT NEVER FAILS THE WRITE. The cell edit has already succeeded and been acknowledged; - an enrichment that could not run is a missing value, not a lost edit, and the run's own report - carries the reason. ⚠ It is also deliberately SYNCHRONOUS and bounded to this one row: a - fan-out here would put a vendor call on the critical path of every keystroke-commit. - """ - import ai_enrich as _ae - - try: - defn = _ut().get(table_key, st=session.runtime) or {} - wanted = _ae.on_change_fields(defn, changed.keys()) - for field in wanted: - # ⭐ W35-T41 / C7 — `user` is the usage ledger's attribution. An on-change run is still - # somebody's edit spending somebody's tokens, so it is booked against the person who - # typed rather than left unattributed. - _ae.run_field(table_key, field["key"], st=session.runtime, rows=[str(pid)], - user=session.uname) - except Exception: # noqa: BLE001 - pass - - -@router.post("/tables/{table_key}/fields/{fkey}/enrich") -def enrich_field(table_key: str, fkey: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Run an AI enrichment column (wave 34, owner ruling R13). Returns the run's own REPORT. - - ⛔ THIS DOOR SPENDS MONEY, so it rides the same wall every other schema write rides - (`_field_or_refuse`) rather than a looser one of its own. A read-only viewer cannot bill the - tenant by opening a grid. - - `{"rows": ["3"]}` is a MANUAL run of exactly those records: a person asked, in front of the - value being replaced, so it skips the `overwrite` policy. An ABSENT `rows` is the automatic - plan, where the policy and the never-overwrite-a-human law both apply. The two are one - function with one flag, not two runners. - - ⚠ THE REPORT IS THE PRODUCT, not a status code. It carries `filled`, `failed`, `skipped` by - reason, `tokens` spent, the provider, per-row errors, and `limit` (R6's second sentence: a - ceiling that stopped the run names its cause and a remedy). A 200 with `filled: 0` and a - populated `skipped` is a correct, informative answer, and the client must render it rather - than treat it as success. - """ - _field_or_refuse(session, table_key, fkey) - import ai_enrich as _ae - - rows = (body or {}).get("rows") - if rows is not None and not isinstance(rows, list): - raise err(400, "bad_rows", "`rows` must be a list of record ids, or absent to run the " - "rows this column's own settings choose") - # The caller's permitted pool, the same one the row doors use. A named row outside it is - # dropped rather than refused: a stale client naming a record that has been deleted or moved - # out of scope should not fail a run over the rows it can legitimately fill. - if rows is not None: - allowed = {str(p) for p in scoped_pids(session, table_key)[0]} - rows = [str(r) for r in rows if str(r) in allowed] - # ⭐ `W34-T54`'s bulk menu is this one field: "Rows never filled" sends `blank`, "All rows" - # sends `always`. Anything else falls back to the column's own saved policy rather than to a - # default, so a typo cannot quietly widen what a run touches. - report = _ae.run_field(table_key, fkey, st=session.runtime, rows=rows, - policy=str((body or {}).get("scope") or "") or None, - # A named row set through THIS door is a person asking. - manual=rows is not None, - # ⭐ W35-T41 / C7 — the usage ledger's attribution. - user=session.uname) - if report.get("problem"): - # A run that could not start at all is not a 200: nothing was attempted, nothing was - # spent, and the reason is actionable (no provider configured, or the wrong column). - raise err(400, "enrich_refused", str(report["problem"])) - return report - - -@router.delete("/tables/{table_key}/fields/{fkey}") -def delete_field(table_key: str, fkey: str, session: Session = Depends(require_session)): - _field_or_refuse(session, table_key, fkey) - if not _ut().delete_field(table_key, fkey, st=session.runtime): - raise err(400, "refused", - "that column could not be removed. A database must keep at least one") - _refresh_relations(session) - return {"deleted": fkey} - - -@router.delete("/tables/{table_key}/rows/{rid}") -def delete_row(table_key: str, rid: str, session: Session = Depends(require_session)): - # ⭐⭐ W31 QA — ONE SNAPSHOT FOR THE WHOLE DELETE, and the owner reported what it cost. - # Owner, verbatim (2026-08-13): *"it still takes forever to delete a record from TT Profile."* - # A single DELETE was FIVE whole-document deep copies before the commit even began — three in - # the guard (`get` + `may_open` + `records_mutable`) and two more inside - # `core.user_tables.delete_row` (`is_user_table` + `records_mutable` again). MEASURED: 2 reads - # cost 45 ms on an 0.8 MB fixture, and tenant #0's `user_tables` document is **28.5 MB** - # (D-185), so the guard alone was seconds of copying to answer questions about one row. - # ⚠ THE LEND IS READ-ONLY AND THE WRITE STILL GOES THROUGH THE REAL RUNTIME — `_Lent` - # `__getattr__`-passes `update` straight to it, and `_drop` runs against the LIVE document - # under the store lock, so a lent snapshot can never be the thing written back. - # ⚠ `flush='sync'` IS DELIBERATELY UNTOUCHED (D-118): nobody spams a delete, and an eventually - # consistent delete is indistinguishable from one that did not work. This makes the guard - # cheap; it does not make the commit optimistic. - lent = _ut().lend(session.runtime) - _records_or_refuse(session, table_key, st=lent) - # ⭐⭐ W36-T21 — THE ROW SCOPE, ON THE DELETE DOOR. `patch_row` has asked this since it was - # written (`pid not in g["pids"]` -> 403) and this door never did, because until now every - # account that could open a `ut_*` database could see every row of it. The moment an - # administrator can row-scope one, "may not SEE row 5" and "may DELETE row 5" become two - # different answers unless this is here — and delete is the one that cannot be undone. - # ⚠ Guarded on `row_scope_applies` so an unscoped account pays nothing: for them the pid set - # is every row and the question has one answer. - import core.perm_scope as _ps - if _ps.row_scope_applies(session.user, table_key): - _pids, _f, _d = scoped_pids(session, table_key) - if not str(rid).isdigit() or int(rid) not in _pids: - raise err(403, "out_of_scope", "that row is not in this database") - try: - ok = _ut().delete_row(table_key, rid, st=lent) - except Exception: - raise err(503, "store_unavailable", "the delete did not land. Try again") - if not ok: - raise err(400, "refused", "rows can only be deleted from user-created databases") - _refresh_relations(session) - return {"ok": True} - - -@router.patch("/tables/{table_key}/rows/{pid}") -def patch_row(table_key: str, pid: int, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Cell edits — the products PATCH on the user-table ctx. Routed through - `core.grid_events.handle_one` so truncation and permission rules stay ONE implementation; - the accepted values are read BACK from the bucket, never echoed from the request.""" - from core import grid_events - - updates = dict(body or {}) - if not updates: - raise err(400, "empty_patch", "no fields to update") - _records_or_refuse(session, table_key) - g = ut_assembly(session, table_key, consume_corrections=False) - if pid not in g["pids"]: - raise err(403, "out_of_scope", "that row is not in this database") - ctx = grid_events.EventCtx( - uname=session.uname, allowed_pids=g["pids"], fields=g["fields"], - admin=session.admin, fallback_ws=None, seen_ids={}, - # ⭐⭐ W36-T21 — the assembly's OWN closure, not an empty set. `g["fields"]` is already - # stripped, but `grid_events` asks `ctx.hidden_keys` by name: a PATCH naming a hidden - # column would otherwise be accepted on a payload that never showed it. - hidden_keys=g.get("hidden") or frozenset(), - st=session.runtime, # R6b (D-16) - scope_key=table_key, table=_ops(session, table_key)) - try: - grid_events.handle_one( - {"id": f"patch:{table_key}:{pid}:{time.time_ns()}", "type": "overlay_patch", - "pid": pid, "updates": updates}, ctx) - except grid_events.StoreUnavailable: - raise err(503, "store_unavailable", - "the tenant store is unavailable. Your change was not saved") - # ⚠ THE READ-BACK IS THE DEFINITION ROW, AND ON A `ut_` SCOPE THAT IS THE WHOLE OF IT. - # - # ⛔ CORRECTED, wave-29 T22 (owner item 2a): this note used to say "THE READ-BACK SPANS BOTH - # STRATA, and it has to (wave 25, C3-A1)" while the two lines under it read exactly one bucket. - # It was true of the wave-25 world it was written in — an ordinary cell landed in the caller's - # OVERLAY and only a PROFILE cell wrote through — and it stayed after `grid_events` began - # routing EVERY accepted cell on a `ut_` scope to `user_tables.patch_cells` - # (`grid_events.py:1979-1984`). There is no second stratum this read is missing; a docstring - # claiming otherwise is what makes the next reader look for a merge bug that is not here. - # ⚠ Legacy overlay values, written before that routing existed, are still merged for DISPLAY - # by `table_rows` (:587-595) — display only, and deliberately not re-asserted here: `_took` - # asks whether THIS write landed, and this write goes to the definition. - stored = dict(((_ut().get(table_key, st=session.runtime) or {}).get("rows") or {}) - .get(str(pid)) or {}) - accepted = {k: stored.get(k) for k in updates if k in stored} - - def _took(k): - """Did the cell TAKE this write? Normally that is "stored == asked". - - ⚠ A PROFILE COLUMN CANONICALISES, so "stored != asked" is its NORMAL success: `@Nurilab` - and `instagram.com/nurilab` both store `nurilab`. Reporting those as refused would tell - the client to roll back a write that landed. But it cannot simply be exempted either — - a junk handle leaves the OLD value sitting in `stored`, which would then read as - accepted. So the question asked is the exact one: **is what is stored the canonical form - of what was asked?** Anything else is a genuine refusal. - """ - if k not in accepted: - return False - want = str(updates[k]) - if stored.get(k) == want: - return True - pf = _ut().profile_field(table_key, st=session.runtime) - if pf and pf["key"] == k: - handle, ok = _ut().normalize_profile(want, pf["profile"].get("source")) - return bool(ok) and stored.get(k) == handle - return False - - refused = sorted(k for k in updates if not _took(k)) - # ⭐⭐ WAVE-34 (R13) — THE HUMAN-EDIT STAMP, AND IT HAS TO HAPPEN HERE. "Did a person write - # this cell?" is not recoverable from the value afterwards, so the only place to record it is - # the door a person writes through. `ai_enrich_may_write` then refuses to let any automatic - # run overwrite it, whatever the column's `overwrite` policy says. - # ⚠ STAMPED FROM THE CELLS THAT ACTUALLY TOOK, never from what was asked: marking a refused - # write `human` would freeze a cell against the agent on the strength of an edit that never - # landed. `note_human_edit` filters to the enrichment columns itself and is a no-op otherwise. - took = {k: v for k, v in accepted.items() if k not in refused} - if took: - try: - _ut().note_human_edit(table_key, took, pid, st=session.runtime) - except Exception: # noqa: BLE001 - # Provenance is metadata about a write that has already succeeded. Failing the - # request here would tell the user their edit was lost when it was not. - pass - _fire_on_change(table_key, pid, took, session) - out = {"pid": pid, "updates": accepted} - if refused: - out["refused"] = refused - # ⭐ R6: the cells the SERVER changed that the client never typed — the preset cells a - # profile blank cleared. Without this the grid keeps painting a stale follower count under - # an empty handle until something else forces a refetch, which is the NO-BLIP LAW's other - # half: the client may keep only what the server actually took, and must be TOLD what else - # moved. Derived by diffing this row against what was asked for, so it cannot drift from - # whatever the clear rule decides to touch. - also = {k: v for k, v in stored.items() if k not in updates and str(v or "") == ""} - cleared = sorted(k for k in also if k in _ut().PROFILE_PRESET_KEYS) - if cleared: - out["cleared"] = cleared - # ⭐⭐ R9's SECOND RE-ARM DOOR — the one call that makes `engine.clear_gone` live (wave 28, - # amendment A5; SESSION B built and gated the function and correctly declared it INERT until - # this line existed, citing [[flag-shipped-without-its-writer]]). - # - # ⛔ R9 makes a `not_found` handle a TOMBSTONE, not a 30-day backoff: nothing re-buys a dead - # account on a timer any more. Door 1 — correcting the handle — needs no wiring, because the - # verdict is keyed on `(platform, handle)` and a corrected handle simply is not the verdict we - # recorded. THIS is door 2: a human re-typing the SAME handle, which is how somebody says "try - # it again, the account is back". Without this call that person has no way back at all, and - # the failure costs nothing and raises nothing — so no spend-shaped test would ever find it. - # - # ⚠ GATED ON `updates`, NOT ON `accepted`: re-typing the identical value is the whole case this - # door exists for, and a no-op write can be filtered out of `accepted`. What matters is that a - # human touched the handle cell. - # ⚠ NOT a bare `except: pass`. A swallowed AttributeError here would be exactly the optional- - # prop silence this wiring exists to prevent — if the engine ever loses `clear_gone`, that must - # be readable in the log rather than degrade into "the re-arm quietly stopped working". - _pf = _ut().profile_field(table_key, st=session.runtime) - if _pf and _pf["key"] in updates: - try: - import automation_engine as _engine - _engine.clear_gone(session.runtime, table_key, stored.get(_pf["key"])) - except Exception as e: # noqa: BLE001 - print(f"[aios-api] clear_gone failed: {type(e).__name__}: {e}") - _refresh_relations(session) - return out - - -# ── CONTRACT C1 (W36-T20): THE MIRROR READER ────────────────────────────────────────────────── -# ⭐⭐ R6. `core.perm_scope.scoped_table` is the ONE door to any database's rows and it answers for -# a MATERIALISED `ut_*` table entirely on its own — deliberately, so a cold process (E's sandbox -# subprocess, a worker, a gate) that never imported a route still gets the right answer. A -# READ-THROUGH grid is the one arm it cannot serve alone: those ten databases store no rows in the -# tenant document at all, and their rows live in the DuckDB mirror behind `routes_odoo_tables`, -# two layers above `core`. -# -# ⛔ ONE FETCH, NOT A SECOND ONE. This hands C1 the SAME `_read_through_rows` that `scoped_pool` -# and `scoped_pids` already share, so the rows a script sees through C1 are byte-for-byte the rows -# the grid sees — by construction, not by a second query that agrees today (W31-T20's argument, one -# caller further out). -# -# ⚠ AND `TooBigToMaterialise` BECOMES A REPORTED REFUSAL, NEVER A SHORT ANSWER. Standing rule 1's -# second sentence, and the shape is `_PID_SCOPE_LIMIT`'s so nothing downstream needs a second -# vocabulary for it. -def _c1_mirror_rows(table_key, field_keys, st): - """C1's mirror arm: `perm_scope.register_mirror`'s reader over `_read_through_rows`.""" - import core.perm_scope as perm_scope - - try: - return _read_through_rows(table_key, field_keys, rt=st) - except _too_big() as e: - raise perm_scope.Unresolvable( - cause=str(e), - recommendation=_PID_SCOPE_LIMIT["recommendation"], - subject="rows", effect="unresolved") from e - except RuntimeError as e: - raise perm_scope.Unresolvable( - subject="rows", effect="unreadable", cause=str(e), - recommendation="the connector mirror is not ready on this process; retry once the " - "store has finished opening") from e - - -def _register_mirror(): - """Declare the mirror reader to C1. Called at import; returns True once it is registered.""" - import core.perm_scope as perm_scope - return perm_scope.register_mirror(_c1_mirror_rows) - - -_C1_MIRROR = _register_mirror() +"""routes_tables.py — USER TABLES over the wire (wave 18, contract C3-UT). + +The runtime-database primitive: `core/user_tables.py` (wave-9 C6, host-only until now) served +through the SAME grid machinery every other topic rides — `table_store` for the per-user +workspace strata, `aios_grid.workspace_wire` for the wire shape, `core.grid_events` for every +durable write except rows. Rows are the one genuinely new channel: the events seam has no row +event types (the user_tables docstring's `row_add` gate was described, never built), so row +add/delete/patch are REST endpoints here, walled by `user_tables.is_user_table` + +`user_tables.may_open` — a connector-backed table can never accept an invented row. + +TENANCY: every store touch goes through `session.runtime` (the tenant's store handle), so a +Nurilab admin's tables live under Nurilab's prefix/repo, and the isolation gate's proof #3 +covers them for free. + +VISIBILITY is `user_tables.may_open` — creator, admin, or a `core.shares` grant, fail-closed, +applied in `_defn_or_refuse` before any payload is built. + +⭐⭐ WAVE 36 (W36-T21 / OWNER RULING R6) — AND IT IS NO LONGER THE WHOLE WALL, WHICH IS THE POINT +OF THE TICKET. That paragraph used to end *"the per-table wall is the whole wall"*, and it was +true: a `ut_*` database was a BINARY door, so an admin could hand somebody all 31,418 rows of +`ut_odoo_invoices` or none of them, while `customer_data` had per-user row filters and hidden +fields. `perms.py`'s own docstring booked the fix and warned what half a fix looks like — *"a +stored `ut_*` wall would be INERT: the editor would say DENY, the table routes would keep serving, +and nothing anywhere would say so."* + +So THREE things are now true of every row this file serves, and each has one place: + * `perm_scope.may_read` — IF: admin, then an explicit stored `access: false`, then `may_open` + UNCHANGED. Composed, never merged. + * `scoped_pool`/`scoped_pids` — WHICH ROWS: the permanent filter, applied BEFORE `pids` is + taken, exactly where `routes_customers.grid_assembly` applies it. + * `ut_assembly` — WHICH FIELDS: the transitive hidden closure, applied AFTER + `workspace_wire`, on BOTH wires (the field list and the row payload). + +⛔ AND A DOOR THAT CANNOT APPLY ANY OF IT REFUSES RATHER THAN SERVING THE LOT — `_defn_or_refuse`'s +`scope_applied` flag, which defaults to fail-closed precisely because the one caller outside this +file cannot pass it. +""" +import threading +import json +import time + +from fastapi import Body, Depends +from fastapi import APIRouter + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + + +def _ut(): + import core.user_tables as user_tables + return user_tables + + +def _ops(session, table_key, st=None): + """This database's per-user workspace store, bound to the tenant. + + ⭐⭐ W36-T24 / D-214 — `st` LETS ONE ASSEMBLY LEND ITS OWN SNAPSHOT, AND THE DOCUMENT IT SAVES + IS `object_shares`, NOT THE WORKSPACE. MEASURED with a call-counting probe over `ut_assembly` + (D-214's own exit condition): one assembly took **5 whole-document reads for an admin and 6 + for a shared, row-scoped user** — `object_shares` TWICE (three times for a non-creator), + `user_tables` once, and `_table_workspace` twice. + + The `object_shares` repeats come from `grid_events._granted_views` and `_granted_folders`, + which ask `shares.shared_with` for the `view` and the `folder` kind separately, each reaching + the store through `tops.st` — this handle. `user_tables.lend` already memoises exactly those + two buckets for one pass (`_LENDABLE`), so handing the ops object a lend collapses them + without touching `core/shares.py`'s semantics or `grid_events`, which is in no wave-36 fence. + + ⛔ WHY IT IS SAFE ON A PATH THAT ALSO WRITES. `_Lent` serves ONLY `user_tables` and + `object_shares`; `_table_workspace` is not lendable, so every workspace read and write + passes straight through to the runtime, and `__getattr__` forwards `update` regardless. The + assembly performs no `object_shares` write, and `patch_row`'s post-write read-back reads + `session.runtime` directly rather than this handle — contract C5's rule, unbroken. + """ + import core.table_store as table_store + return table_store.make(f"{table_key}_table_workspace", + st=st if st is not None else session.runtime) + + +#: WAVE 27 item 2 (contract C2 / amendment A2) — the relation refresh is COALESCED and runs OFF +#: the request path. `{tenant: True}` while a pass is queued; the worker holds the lock. +_REL_LOCK = threading.Lock() +_REL_DIRTY = {} +_REL_RUNNING = {} + + +def _refresh_relations(session): + """Mark this tenant's Links/Rollups stale and refresh them AFTER the response, once. + + ⛔ WHY THIS IS NOT A DIRECT CALL ANY MORE, and the numbers are the argument. The owner's + item 2 was "adding a new record visually takes too long, I need to be able to spam it", and + this function was the largest single cost inside `POST /tables/{key}/rows`: + `engine.refresh_relations` DEEP-COPIES every table and every row in the tenant before it can + decide whether anything needs doing (`automation_engine.py:9383-9388`), and when something + does it commits with `flush="sync"` — a store round trip, i.e. an HF Dataset commit on the + default backend. Every added record paid a whole-tenant snapshot plus a synchronous commit + before the 201 came back. + + ⛔ AND BACKGROUNDING ALONE WOULD HAVE BEEN A WORSE BUG. Twenty rapid adds would queue twenty + whole-tenant snapshots, each one re-reading a store the previous one just wrote — the spam + the item asks us to support is exactly the load that would melt it. So this COALESCES: at + most one pass runs, and at most one is queued behind it. + + ⚠ THE DIRTY FLAG IS THE LOAD-BEARING PART, not the lock. A write landing WHILE a pass is in + flight must still get a pass afterwards, because the in-flight one snapshotted before that + write existed. Without the flag the LAST add in a burst is precisely the one whose rollups + never update — the failure nobody would notice until a total was quietly wrong. + + Eventual consistency is the accepted trade and was already the documented posture: the tick + repairs materialised cells regardless, and answering 503 here would invite the browser to + repeat a mutation that already succeeded. + """ + tenant = str(getattr(session, "tenant", "") or "") + rt = session.runtime + with _REL_LOCK: + _REL_DIRTY[tenant] = True + if _REL_RUNNING.get(tenant): + return # a worker is live; it will see the flag and loop + _REL_RUNNING[tenant] = True + + def _worker(): + import automation_engine as engine + try: + while True: + with _REL_LOCK: + if not _REL_DIRTY.get(tenant): + _REL_RUNNING.pop(tenant, None) + return + _REL_DIRTY.pop(tenant, None) + try: + engine.refresh_relations(rt, log=lambda *_args: None) + except Exception as exc: # noqa: BLE001 + # The source write already landed and the response is already sent. Log and + # let the tick repair it; never retry in a tight loop. + print(f"[tables] relation refresh deferred: {type(exc).__name__}: {exc}") + finally: + # Belt for an unexpected raise on the bookkeeping itself: a tenant left marked + # RUNNING would never refresh again for the life of the process. + with _REL_LOCK: + if _REL_RUNNING.get(tenant) and not _REL_DIRTY.get(tenant): + _REL_RUNNING.pop(tenant, None) + + threading.Thread(target=_worker, daemon=True, + name=f"rel-refresh:{tenant or 'default'}").start() + + +def _defn_or_refuse(session, table_key, st=None, defs_only=False, scope_applied=False): + """The per-table wall: 404 for a key that does not exist, 403 for one this session may not + open. 404-before-403 leaks nothing useful — ut keys are guessable slugs, and 'exists but + not yours' is exactly what may_open is for. + + ⭐⭐ W36-T21 / R6 / CONTRACT C1 — `scope_applied` IS THE HALF THAT MAKES THE WALL NON-INERT, + AND IT DEFAULTS TO FAIL-CLOSED ON PURPOSE. + + `perms.py`'s own docstring described exactly the defect this parameter prevents: *"a stored + `ut_*` wall would be INERT — the editor would say DENY, the table routes would keep serving, + and nothing anywhere would say so."* So a door that will apply C1's row/field wall to what it + is about to serve says so HERE, in writing, and a door that will not is REFUSED for any + principal carrying a wall on this database (`perm_scope.wall_declared`). + + ⛔ THE DEFAULT IS `False` BECAUSE THE ONE CALLER OUTSIDE THIS FILE CANNOT PASS IT. + `routes_odoo_tables`' windowed rows route calls this guard and then builds its own SQL — it + has no way to apply a row filter or a hidden-field closure, and it is in no wave-36 fence. An + opt-OUT default would have left that door silently serving a walled account the whole table, + which is the INERT wall by another route. Opting IN means a door added tomorrow is refused + until somebody has thought about it ([[default-must-pass-its-own-guard]]). + + ⚠ AND IT COSTS NOTHING FOR EVERYBODY WITH NO WALL — which today is every account in every + tenant, because no `ut_*` wall has ever been storable. `wall_declared` is False for an admin + and False for a record with no entry, so this branch cannot fire until an administrator + deliberately stores one. + + ⭐ `st` LETS A CALLER LEND THE SNAPSHOT IT IS ALREADY HOLDING (W31 QA), the same `lend()` the + nav and `/tables` use since W31-T10/T12. The WALL is untouched — the same `may_open`, asked + about the same table — it is simply not re-reading a 28.5 MB document to ask it. + + ⭐⭐ W33-T01 (D-213) — `defs_only=True` MAKES THAT READ A PROJECTION, AND IT IS OPT-IN PER CALL + SITE ON PURPOSE. This wall asks two questions ("does the key exist", "may this session open + it") and neither has ever read a row, but three of its ten callers go on to read `rows` OFF THE + DEFINITION THIS RETURNS — so a blanket swap would turn `scoped_pool`, `scoped_pids` and + `table_footprint` into `KeyError`s on every materialised table. The flag is therefore the + CALLER's claim about what it will do next, not a global setting; each opt-in below is annotated + with why it can make that claim ([[reuse-and-delete-are-hypotheses]]). + + ⛔ **`defs_only` IS FOR READS.** The six write walls (`_records_or_refuse`, `patch_shared_cell`, + `delete_shared_field`, `patch_table`, `delete_table`, `import_rows`) deliberately do NOT pass + it: a projected snapshot must never reach a post-write read-back (contract C5), and the + pre-write lend note below is the same argument one layer down. + + ⚠ **WHAT COMES BACK IS A `store._Projected`, WHICH IS THE EVIDENCE, NOT AN IMPLEMENTATION + DETAIL.** `all_defs` falls back to the whole read on ANY failure, so "the answer was right" + proves nothing about which read produced it. `core.store.is_projected(defn)` is how a gate + asserts the fast path was TAKEN. + """ + ut = _ut() + # ⭐⭐ W31 QA — THE SWEEP, AND IT IS ONE LINE BECAUSE IT IS DONE HERE RATHER THAN PER ROUTE. + # Owner: *"this is just one case, I need you to check and apply the fix everywhere too."* + # This guard has TEN direct call sites (counted 2026-08-14; the note said FOURTEEN, which was + # the route count, not the caller count) and cost TWO whole-document deep copies at every one + # (`get`, then `may_open`) — on tenant #0 that is 2 x 28.5 MB (D-185) to answer two questions + # about ONE table. Lending here fixes every caller at once, including the eight routes the + # sweep enumerated (`PATCH /shared/{pid}` · `DELETE /shared/fields/{k}` · `GET|POST /rows` · + # `POST /rows/import` · `POST|PATCH /fields` · `PATCH /rows/{pid}`). + # ⛔ WHY IT IS SAFE HERE AND WOULD NOT BE IN THE ROUTES: a lend is a PRE-WRITE snapshot. The + # guard runs before any mutation and returns only the DEFINITION, so the snapshot never + # survives to serve a read-back. Blanket-replacing `st=session.runtime` inside the routes + # would hand that stale snapshot to `patch_row`'s and `add_row`'s post-write read-back, which + # is [[refetch-eats-its-own-write]] with the sign flipped — a write that reports the value it + # replaced. The remaining in-route reads are deliberately untouched and booked instead. + if st is None: + st = ut.lend_defs(session.runtime) if defs_only else ut.lend(session.runtime) + defn = ut.get(table_key, st=st) + if not defn: + raise err(404, "unknown_table", "that database does not exist") + # ⭐⭐ W36-T21 — ONE evaluator for the IF question, and it CALLS `may_open` rather than + # replacing it. `perm_scope.may_read` is admin -> an explicit stored `access: false` -> + # `user_tables.may_open`, unmodified. Two questions stay two questions: `may_open` still + # decides IF the database is visible, C1 decides WHICH rows and fields. + import core.perm_scope as perm_scope + if not perm_scope.may_read(session.user, table_key, st=st): + raise err(403, "forbidden", "that database belongs to another user") + if not scope_applied and perm_scope.wall_declared(session.user, table_key): + # R6's second sentence: a limit that cannot be met is a SENTENCE naming the cause and a + # fix, never a short answer — and here the short answer would be the WHOLE database. + raise err(409, "scope_not_applied", + "an administrator has restricted which rows and columns of this database you " + "may see, and this view cannot apply that restriction. Open the database from " + "the navigation, where the restriction is applied, or ask an administrator to " + "remove it") + return defn + + +def _records_or_refuse(session, table_key, st=None): + """The human record-write wall for a database the automation engine owns.""" + # One lend for BOTH questions this wall asks — the definition wall and the record-mode wall — + # so the two are answered from one read instead of two. Same reasoning as `_defn_or_refuse`. + st = st if st is not None else _ut().lend(session.runtime) + # ⭐ W36-T21 — `scope_applied=True` because every row write behind this wall is bounded by a + # SCOPED pid set of its own: `patch_row` refuses a pid outside `ut_assembly`'s `pids`, and + # `delete_row` asks the same question just below. Refusing here instead would make a walled + # database READ-ONLY rather than row-scoped, which is a different product. + defn = _defn_or_refuse(session, table_key, st=st, scope_applied=True) + if not _ut().records_mutable(table_key, st=st): + raise err(403, "records_read_only", + "records in this automation-owned database are read-only. Add Instagram " + "handles in a Profile database and let enrichment populate this database") + return defn + + +#: ⛔ THE PER-USER CANDIDATE WALL IS RETIRED (WAVE 27, DEBT D-72). **THE TENANT IS THE UNIT, NOT +#: THE USER** — one profile is ONE row, and everyone who may open the database sees all of it. +#: +#: WHY IT HAD TO GO, and it is not a preference: wave 26's R4 made the candidate identity +#: `(platform, handle)` and `_merge_candidates` stamps only the FIRST finder, later finders never +#: overwriting. Combined with a wall that then showed a non-admin only `created_by == me`, the +#: two were SILENT DATA LOSS — the second finder's row was merged away into the first finder's, +#: and the wall then hid the survivor from the person who just found it. They searched, they +#: paid, and the screen said nothing arrived. +#: +#: ⚠ AND THE PAIR WAS MUTUALLY MASKING, which is why it stayed green for a wave: the wall named +#: `ut_ig_candidates`, and a READ-ONLY census of all four tenant stores +#: (`ops/w26_candidate_census.py`) proved that table exists in NO tenant — since wave 25's R2 the +#: write target is whatever database the user points the Create-record action at. So the wall +#: governed a table nobody writes, and fixing EITHER half alone would have armed the other +#: ([[defects-that-mask-each-other]]). +#: +#: The register offered two exits and R4 already implied this one. Restoring per-user visibility +#: instead would have required R4's merge to stop crossing users — a bigger change, against the +#: ruling, to bring back a wall that never governed anything real. +#: +#: ⛔ DO NOT RE-ADD THIS BY INFERRING THE RULE FROM A `created_by` COLUMN. Every +#: automation-written table has one (a scraped row says `automation`), so inference would hide +#: every scraped row from every non-admin — the same disappearance defect, one table wide instead +#: of one table narrow. `created_by` survives as W26/R4's informational "Found by" stamp ONLY. + + +def _too_big(): + """`routes_odoo_tables.TooBigToMaterialise`, imported lazily — one name, two policies below.""" + import routes_odoo_tables + return routes_odoo_tables.TooBigToMaterialise + + +def _read_through_rows(table_key, field_keys, rt=None): + """The mirror's rows for one read-through grid, projected to this table's declared columns. + + ⭐⭐ W31-T45 / D-169 — `rt` IS THE SESSION'S TENANT RUNTIME AND IT IS PASSED THROUGH (D's ask, + `mailbox/D.md` D-2). `_defn_or_refuse` above answers *"may this SESSION open this DATABASE"*; + the guard `whole_pool` fires on `rt` answers a DIFFERENT question — *"is the DuckDB file this + process has open THIS TENANT's"* — and R2 gives GTM Lab connected tables in its own document, + which is exactly the shape that satisfies the first wall while failing the second. + `datastore.ro_con()` reads a process-global `DB_PATH` and one Space process serves every + tenant, so the two walls are not substitutes for each other. + + ⛔ ONE FETCH, TWO POLICIES — and the split is the whole of W31-T20. Both `scoped_pool` (which + owes a caller every row) and `scoped_pids` (which owes only the row SET) reach the mirror + through this function, so the pid set the cheap path answers with is IDENTICAL to the pool's + by construction rather than by a second query that agrees today. A pid-only `SELECT` would be + cheaper and would also be a SECOND statement of what a row of this table is + ([[one-question-two-normalizers]]) — the two callers differ in what they do with the REFUSAL, + never in how they ask. + + Raises `TooBigToMaterialise` when the population exceeds one window; the caller decides + whether that is a 409 or an unresolved pid scope. + """ + import routes_odoo_tables + + rows_src = [{k: v for k, v in r.items() if k in field_keys or k == "pid"} + for r in routes_odoo_tables.whole_pool(table_key, rt=rt)] + rows_src.sort(key=lambda r: r["pid"]) + return rows_src + + +#: ⭐⭐ W31-T20 / D-174 — R6's SECOND SENTENCE FOR A PID SCOPE THAT CANNOT BE RESOLVED. +#: +#: Wave 30 un-capped the ROW door and left the WORKSPACE envelope, so `GET /workspace?scope= +#: ut_odoo_gl_lines` answered `409 window_required` six times out of six on the live deploy and the +#: grid painted an ERROR PAGE with a Retry button — a whole shipped feature nobody could open. +#: The cause: an envelope needs no row, but it asked for every one of 963,783 of them to derive a +#: pid set it uses for exactly two things (cohort membership and a shared view's `memberPids`). +#: +#: ⛔ SO THE ENVELOPE STOPS ASKING, AND SAYS SO. An empty pid set is not "no rows" — it is +#: "membership is unresolved on this grid", which is a different claim and has to be made out +#: loud, on the wire, or it is the silent truncation R6 is actually about. Both consequences are +#: fail-closed: a stored cohort's members are reported MISSING by `aios_grid.workspace_wire` +#: rather than silently dropped, and any WRITE that names a pid is refused by `routes_grid`. +_PID_SCOPE_LIMIT = { + "subject": "pids", "effect": "unresolved", + "recommendation": "cohorts and shared-view membership are resolved per page on this grid; " + "filter and read it a window at a time (`/odoo-tables/{key}/rows`), where " + "every total is a SQL count over the whole table", +} + + +def scoped_pool(session: Session, table_key: str, st=None): + """`(pids, rows_src, fields_base, defn)` — THE USER-TABLE WALL, on its own. + + ⭐ W33-T03 (D-214) — `st` LETS A CALLER LEND THE DOCUMENT IT IS ALREADY HOLDING, exactly as + `_defn_or_refuse` has since W31 QA. It is passed straight through to that wall and nowhere + else, so the permission question is answered by the same code against the same document. + ⛔ READ CALLERS ONLY. A lend is a PRE-WRITE snapshot; handing one to a path that writes and then + reads back is [[refetch-eats-its-own-write]] with the sign flipped (contract C5). + + `routes_products.scoped_pool`'s sibling, extracted for the same reason (wave 19, item 12): a + caller that only needs "which rows of this database may this session touch" — record comments + — must ask through `_defn_or_refuse` (404/403, the whole wall) rather than growing a second + idea of what a user table's pool is. + + ⭐⭐ W36-T21 / R6 — THE ROW WALL RUNS HERE NOW, ON EVERY DATABASE, and the position is the + whole of it: BEFORE `pids` is taken. `routes_customers.grid_assembly` says the same thing in + the same words for `customer_data` — everything downstream is bounded by that frozenset + (`allowed_pids` for the workspace, cohort membership, every write door's pid check), so + scoping here means a row this account may not see never enters ANY of them, rather than being + filtered out of one payload and surviving in another. + + ⚠ THE FIELD HALF IS NOT HERE, and that is parity rather than an omission: the hidden closure + must cover the user's own `custom_`/`measure_` columns, which do not exist until + `workspace_wire` has run. `ut_assembly` applies it there, exactly where `grid_assembly` does. + """ + defn = _defn_or_refuse(session, table_key, st=st, scope_applied=True) + fields_base = [dict(f) for f in (defn.get("fields") or [])] + field_keys = {f["key"] for f in fields_base} + # ⭐⭐ W30-T31 / D-87 — A READ-THROUGH DATABASE HAS NO ROWS HERE, SO THEY COME FROM THE MIRROR. + # + # This is the one function that turns "what is stored" into "what this session may see", which + # is exactly why the read-through arm belongs HERE and nowhere else: the rows route, the events + # route, the comments wall and the assembly all reach rows through it, so they all convert + # together or not at all. ⛔ Reading `defn["rows"]` for such a table would find `{}` and serve + # an EMPTY GRID — correct-looking, wrong, and silent. + # ⚠ The wall above has already run. This adds no scope of its own and takes none away. + import core.perm_scope as perm_scope + if not _ut().materialises(table_key, st=session.runtime, defn=defn): + try: + rows_src = _read_through_rows(table_key, field_keys, rt=session.runtime) + except _too_big() as e: + # R6's second sentence: a limit that cannot be met is a SENTENCE, never a short grid. + # ⚠ THE ROWS PATH STILL REFUSES, AND THAT IS CORRECT (W31-T20 changed the ENVELOPE, not + # this): a caller that asked for every row of a 963,783-row grid cannot be served a + # short one. `scoped_pids` below takes the same refusal and answers a different + # question with it, because an envelope needs no row. + raise err(409, "window_required", str(e)) + except RuntimeError as e: + raise err(503, "store_not_ready", str(e)) + rows_src = perm_scope.apply_row_scope(rows_src, session.user, table_key, fields_base) + return frozenset(r["pid"] for r in rows_src), rows_src, fields_base, defn + rows_src = [] + for rid, row in (defn.get("rows") or {}).items(): + if not str(rid).isdigit(): + continue + # WAVE 27 / D-72: no per-row OWNER filter — the row's creator is not a permission. What + # runs below is a different thing entirely: the permanent filter an ADMIN declared for + # this account (W36-T21 / R6), the same one `grid_assembly` has applied to `customer_data` + # since wave 15. A row this session can reach is a row the tenant owns AND the wall admits. + r = {k: v for k, v in (row or {}).items() if k in field_keys} + r["pid"] = int(rid) + rows_src.append(r) + rows_src.sort(key=lambda r: r["pid"]) + # ⭐⭐ W36-T21 — `permits()`, not `matches()`: an unanswerable permanent filter DENIES rather + # than being ignored. Evaluated against the DECLARED contract (`fields_base`), which is what + # `routes_admin._clean_perms` validates a stored filter against, so the two cannot disagree + # about what a column is. + rows_src = perm_scope.apply_row_scope(rows_src, session.user, table_key, fields_base) + return frozenset(r["pid"] for r in rows_src), rows_src, fields_base, defn + + +@router.patch("/tables/{table_key}/shared/{pid}") +def patch_shared_cell(table_key: str, pid: int, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Write a cell into the TENANT-WIDE overlay — the product door `core/shared_overlay.py` has + been waiting for since it shipped (W29-T62, wave 30 T28). + + ⭐ WHY A SEPARATE STRATUM AT ALL, restated because it is the whole feature and it is not + "sharing would be nice": a user-created column and its values live PER USER, so a shared view + filtering on one names a column other accounts do not have — and an unknown column is an + INACTIVE condition in the tri-state engine, which IGNORES it and therefore WIDENS. The buy + list would silently show the whole catalogue to everyone but its author. A column whose value + is the same for every reader is the precondition for editing it at all. + + ⛔ THE WALL IS `_defn_or_refuse`, AND THE STRATUM IS NOT ONE. `shared_overlay` refuses no + reader and no writer by design; "may this session open this surface" is answered HERE, where + the session is. Do not push the question down there. + ⚠ A tenant-wide write is not a private one: every account that may open this database sees it. + That is the point, and it is why this door declares the column too — a value with no + definition is a cell nobody can find. + """ + body = body if isinstance(body, dict) else {} + key = str(body.get("field") or "").strip() + if not key: + raise err(400, "bad_request", "a field key is required") + _defn_or_refuse(session, table_key) + from core import shared_overlay + import core.perm_scope as perm_scope + if not shared_overlay.is_shared(table_key, key, st=session.runtime): + # ⭐⭐ W38-T16 / R7 — A COLUMN IS BORN OWNED AND PRIVATE, AND THIS IS THE PRODUCT CHANGE. + # Until now this door minted a column every account in the tenant could read, with no way + # to say who. `granted` marks it as governed by `core.shares` (see + # `perm_scope.FIELD_GRANT_MARK` for why an explicit marker and not the absence of a grant + # record), and the grant record below claims it for the creator. + # + # ⚠ THE EMPTY ENTRY LIST IS THE POINT, NOT A PLACEHOLDER. `set_grants` keeps a record that + # has an owner and no entries, so "shared with nobody" is STORED and is a different fact + # from "never shared" — the same explicit-resolution shape `perms_v` uses. Without the + # owner the column would be unmanageable: `may_administer` fails closed on an ownerless + # record, so nobody could ever share or re-share it. + # ⚠ COLUMNS THAT PREDATE THIS CARRY NO MARK and stay tenant-wide, unchanged. Reading + # their absence as "granted to nobody" would blank every existing shared column at once. + shared_overlay.put_field(table_key, key, { + "key": key, "label": str(body.get("label") or key), "source": "overlay", + "type": str(body.get("type") or "text"), "shared": True, + perm_scope.FIELD_GRANT_MARK: True, + "createdBy": session.uname}, st=session.runtime) + try: + import core.shares as shares + shares.set_grants("field", shares.field_oid(table_key, key), [], + owner=session.uname, st=session.runtime) + except Exception: # noqa: BLE001 + # ⛔ THE MARK IS ALREADY WRITTEN, SO A FAILED CLAIM FAILS **CLOSED**: the column is + # governed and nobody holds a grant, i.e. only an admin sees it. That is recoverable + # (an admin can share it) and the other order is not — a marked column with a claim + # that landed first and a definition that did not would be a grant on nothing. + pass + try: + # ⚠ `put_cell`, not `put_cells` — this door writes exactly ONE cell, and the singular is + # the API that says so. It delegates to the plural, so both stay reachable through the one + # caller; before this, the singular had no caller at all and `verify_reachability` LENS 2 + # named it (the same lens that found `drop_field` had no door either). + stored = {key: shared_overlay.put_cell(table_key, pid, key, body.get("value"), + st=session.runtime)} + except ValueError as e: + # A non-scalar RAISES in the stratum rather than being dropped; relay it as the answer. + raise err(400, "bad_value", str(e)) + return {"ok": True, "pid": pid, "cells": stored, + "fields": list(shared_overlay.fields(table_key, st=session.runtime))} + + +@router.delete("/tables/{table_key}/shared/fields/{field_key}") +def delete_shared_field(table_key: str, field_key: str, + session: Session = Depends(require_session)): + """Remove a TENANT-WIDE column and every value in it. + + ⛔ WHY THIS EXISTS AT ALL, said plainly: W30-T28 shipped the door that CREATES a shared column + and none that removes one, so a column anybody added was permanent for the whole tenant. The + reachability gate found it from the other end — `shared_overlay.drop_field` was complete, + correct, gated, and callable by nothing but its own gate ([[reachable-is-not-the-same-as-built]]). + + ⛔ AND THIS ONE IS CREATOR-OR-ADMIN, WHICH THE WRITE DOOR IS NOT. Writing a cell changes a + value; dropping the column deletes that value for EVERY account at once, so it is the + destructive-op wall this repo already uses for a database delete — not `editRole`, which + governs renaming and is not a value wall ([[schema-role-is-not-a-value-wall]]). + ⚠ `createdBy` is stamped by the write door above; a column stored before that stamp existed + is admin-only, which is the safe direction. + """ + _defn_or_refuse(session, table_key) + from core import shared_overlay + defn = (shared_overlay.fields(table_key, st=session.runtime) or {}).get(str(field_key)) + if not defn: + raise err(404, "unknown_field", "that column is not a shared column on this database") + owner = str(defn.get("createdBy") or "") + if not session.admin and owner != session.uname: + raise err(403, "forbidden", + f"a tenant-wide column can be removed by its creator or an admin. This one " + f"was added by {owner or 'somebody else'}, and dropping it would delete the " + f"value for every account") + dropped = shared_overlay.drop_field(table_key, str(field_key), st=session.runtime) + # ⭐⭐ W38-T16 — THE GRANTS DIE WITH THE COLUMN, which is `shares.drop_objects`' whole reason + # for existing (wave 21, C3): a deleted object's grants would otherwise serve a ghost id into + # every receiver's "Shared with me" forever, and the ghost 404s on open. It matters twice as + # much for a field, because `drop_field` scrubs the CELLS so the key can be re-used — and a + # surviving grant record would silently re-arm on the next column that took the name. + # ⚠ R8 / D-172 IS UNCHANGED ABOVE: delete is creator-or-admin. This ticket did not widen it. + try: + import core.shares as shares + shares.drop_objects([("field", shares.field_oid(table_key, str(field_key)))], + st=session.runtime) + except Exception: # noqa: BLE001 + pass + return {"ok": True, "dropped": bool(dropped), + "fields": list(shared_overlay.fields(table_key, st=session.runtime))} + + +def scoped_pids(session: Session, table_key: str, limits=None, st=None): + """`(pids, fields_base, defn)` — the SAME wall and the SAME row set as `scoped_pool`, without + building a row. + + ⭐⭐ WAVE 30 / W30-T30 — THIS IS WHY ONE HIDE-FIELDS CHECKBOX WAS EXPENSIVE. A view write + (`view_upsert`) reaches `grid_events_route`, which built a FULL assembly purely to validate + it: `scoped_pool` allocates a fresh dict per row and then sorts them — ~33k order rows, on + every toggle — and the six keys the events route actually reads from that assembly + (`fields`, `pids`, `measures`, `measure_sets`, `lists`, `views`) contain no row at all. + `rows_src` was computed and discarded. + + ⛔ THE PID SET IS IDENTICAL, NOT MERELY EQUIVALENT, and that is the whole safety argument: + `scoped_pool` derives its pids as `frozenset(r["pid"] for r in rows_src)` over exactly the + row ids that pass `str(rid).isdigit()`, which is this comprehension with a dict build in the + middle. The row WALL is unchanged — a narrower or wider set here would be a permission + change, and this is a performance change. + + ⚠ It does NOT make the write cheap on its own: `_defn_or_refuse` still costs a whole-document + read, which is D-87 and W30-T31. This removes the row pass. + ⭐ CORRECTED 2026-08-14 (W33-T01): that sentence said **two** deep copies (`ut.get` then + `may_open`) and had been stale since W31 QA taught the wall to `lend()` — the two questions + have shared ONE read since `routes_tables.py`'s lend line. And as of this ticket the read is a + PROJECTION on the read-through arm, so the sentence is now true only of the materialised one. + Booked because a stale performance note is how a wave re-fixes something twice + ([[stale-baseline-unreadable-deltas]]). + + ⭐⭐ W31-T20 / D-174 — `limits` IS AN OUT-PARAMETER, AND IT IS THE POINT OF THE TICKET. Pass a + list and this function APPENDS R6's sentence to it when the pid set could not be resolved (a + read-through grid whose population exceeds one window). The pid set is then EMPTY, and every + consumer of an empty pid set is fail-closed — but "fail-closed and unannounced" is exactly the + silent limit R6's second sentence forbids, so a caller that renders an envelope or admits a + write is expected to carry the sentence through. Omitting the list means the caller accepts an + unannounced empty scope, which is only ever right for a caller that does not use the pids. + """ + # ⭐⭐ W33-T01 / D-213 — THE DATABASE-SWITCH PATH, AND IT STARTS ON A PROJECTION. + # + # `GET /workspace?scope=` reaches here through `ut_assembly(with_rows=False)` + # (`routes_grid.py`'s ut_ branch), which is what a person is waiting for when they click a + # database in the nav flyout: 1.8-7.3 s live for a 3-6 KB payload, of which one whole-document + # read is ~703 ms warm and 20.6 s cold. This function reads `fields` and (below) `readThrough` + # off the definition — no row — so the WALL can be answered from the 0.1% projection. + # + # ⛔ IT IS THE TRAP ON THIS BOARD, SO IT IS SAID TWICE: this function is NAMED and DOCUMENTED + # as the rows-free twin of `scoped_pool` and the materialised arm below still reads `rows`. + # The opt-in is therefore CONDITIONAL, and the condition is `materialises`, which reads + # `readThrough` — a definition key, safe under the projection, and already lent the defn so it + # costs no read of its own. + import core.perm_scope as perm_scope + # ⭐ W36-T24 / D-214 — `st` LETS THE ASSEMBLY LEND ITS OWN PASS, exactly as `scoped_pool` has + # since W33-T03. ⛔ It must be a PROJECTED lend (`lend_defs`), not a whole one: the saving + # D-213 bought on the database-switch path is that this wall answers from the 0.1% document, + # and handing it `lend()` would quietly take that back while looking like an optimisation. + defn = _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True) + fields_base = [dict(f) for f in (defn.get("fields") or [])] + # ⚠ W30-T31: on a read-through database the stored `rows` is `{}` by construction, so the + # comprehension below would answer an EMPTY pid set — and the promise this function makes is + # that its set is IDENTICAL to `scoped_pool`'s, not merely cheaper. It reaches the mirror + # through the SAME fetch that function uses rather than growing a second idea of the row set; + # the saving W30-T30 bought stays on every materialised table, which is all of the big ones. + if not _ut().materialises(table_key, st=session.runtime, defn=defn): + try: + rows = _read_through_rows(table_key, {f["key"] for f in fields_base}, + rt=session.runtime) + rows = perm_scope.apply_row_scope(rows, session.user, table_key, fields_base) + return frozenset(r["pid"] for r in rows), fields_base, defn + except _too_big() as e: + # ⛔ THE REFUSAL BECOMES AN ANSWER HERE, WHICH IT MUST NOT ON THE ROWS PATH. Turning + # this into a 409 is what made both line grids unopenable: the envelope was refused + # over rows it never renders. The scope is empty and SAID to be empty. + if limits is not None: + limits.append({**_PID_SCOPE_LIMIT, "cause": str(e)}) + return frozenset(), fields_base, defn + except RuntimeError as e: + raise err(503, "store_not_ready", str(e)) + # ⛔⛔ MATERIALISED: THE PID SET *IS* `rows`, SO THIS ARM TAKES THE WHOLE READ — the projection + # above cannot serve it and would raise rather than answer `{}` (that is the whole design of + # `_Projected`). The wall has already passed on the projected document, so this re-reads the + # DEFINITION and does not re-ask `may_open`: re-walling would be a second, differently-shaped + # answer to a question already answered, which is how two ideas of ownership got into this file + # once before (see `may_open`'s own note in `core/user_tables.py`). + # + # ⚠ THE HONEST COST, STATED RATHER THAN BURIED: a materialised table now pays the projection + # PLUS the whole read — ~1.4 ms on top of ~703 ms on tenant #0, i.e. 0.2%. The pid set, the + # wall and the returned shape are byte-for-byte what they were; only tenant #0's ten + # read-through databases (every one of them, which is why the switch was slow) skip the big + # read entirely. + whole = _ut().get(table_key, st=session.runtime) + if whole is None: + # Between the wall and here the table was deleted by another request. Same refusal the + # wall gives, rather than an empty pid set nobody can distinguish from an empty table. + raise err(404, "unknown_table", "that database does not exist") + # ⭐⭐ W36-T21 — AND THE ROW WALL, WHICH IS WHY THIS ARM CAN NO LONGER ALWAYS SKIP THE ROWS. + # + # ⛔ THE PROMISE THIS FUNCTION MAKES IS THAT ITS PID SET IS **IDENTICAL** TO `scoped_pool`'s, + # not merely cheaper. `scoped_pool` now narrows its rows by the permanent filter before taking + # pids, so a set built here from the raw row ids would be WIDER — and every consumer of these + # pids (the workspace envelope, cohort membership, `patch_row`'s scope check) would admit rows + # the read door refuses. That is two ideas of one row set, which is the exact defect class + # `may_open`'s own wave-20 note records ([[one-question-two-normalizers]]). + # + # ⭐ AND W30-T30's SAVING SURVIVES FOR EVERYBODY IT WAS FOR. `row_scope_applies` is False for + # an admin and for every record with no declared filter, which is every account in every + # tenant today — those callers take the id comprehension exactly as before and build no row. + # Only a principal an administrator has actually row-scoped pays the pass, and for them the + # alternative is not "cheaper" but "wrong". + rows = whole.get("rows") or {} + if not perm_scope.row_scope_applies(session.user, table_key): + return frozenset(int(rid) for rid in rows if str(rid).isdigit()), fields_base, whole + keys = {f["key"] for f in fields_base if f.get("key")} + scoped = perm_scope.apply_row_scope( + [{**{k: v for k, v in (row or {}).items() if k in keys}, "pid": int(rid)} + for rid, row in rows.items() if str(rid).isdigit()], + session.user, table_key, fields_base) + return frozenset(r["pid"] for r in scoped), fields_base, whole + + +def ut_write_ctx(session: Session, table_key: str): + """The g-dict a WRITE needs — same keys as `ut_assembly`, no rows. + + Returns the six keys `routes_grid.grid_events_route` reads, so the events route consumes this + or a full assembly interchangeably. `rows_src` is `[]` on purpose rather than absent: a caller + that starts needing rows should fail on an empty list it can see, not on a KeyError. + """ + import aios_grid + from core import grid_events + + limits = [] + # ⭐ W36-T24 / D-214 — ONE lend for the whole pass, so the wall and the grant legs stop + # reading `object_shares` once each. Projected, because this ctx reads no row either. + lent = _ut().lend_defs(session.runtime) + pids, fields_base, defn = scoped_pids(session, table_key, limits=limits, st=lent) + # ⭐⭐ W36-T21 — THE FIELD WALL ON THE **WRITE** CTX, and it is not a copy of the read one. + # `grid_events` refuses a hidden key by asking `ctx.hidden_keys` (`grid_events.py:1802` and + # `:2021`); this ctx passed `frozenset()`, so a hidden column was hidden on the READ and fully + # writable on the EVENTS transport — a wall on one wire and not the other is the shape + # `strip_row`'s own note warns about, with the sign flipped. + hidden = _ut_hidden(session, table_key, fields_base, st=lent) + ctx = grid_events.EventCtx( + uname=session.uname, allowed_pids=pids, fields=[], hidden_keys=hidden, + admin=session.admin, fallback_ws=None, seen_ids={}, st=session.runtime, + scope_key=table_key, table=_ops(session, table_key, st=lent)) + ws = grid_events.table_workspace(ctx, allowed_pids=pids, consume_corrections=False) + workspace, fields, views, lists = aios_grid.workspace_wire( + ws, session.uname, set(pids), defs={}, scope_key=table_key, storage_key="", + fields_base=fields_base) + # ⭐⭐ W38-T16 — THE SHARED COLUMNS JOIN THE **WRITE** CTX'S CONTRACT TOO, and for the reason + # W36-T21 gave one wall over: `grid_events` refuses a key by asking `ctx.hidden_keys`, so a + # column missing from this list is a column the events transport does not know it must + # refuse. The read door narrowing a name the write door has never heard of is the same + # one-wire wall with the sign flipped. + fields = _ut_shared_fields(session, table_key, fields) + fields, _rows, hidden = _ut_field_wall(session, table_key, fields, [], st=lent) + return {"rows_src": [], "pids": pids, "ws": ws, "workspace": workspace, + "fields": fields, "views": views, "lists": lists, "hidden": hidden, + "derived": aios_grid.cohort_cells(lists), + # ⭐ W37-T12 — the OFFER, because `routes_grid` reads `g["measures"]` as the ADMISSION + # list for `clean_measure_field`: without it a Metric create on this surface is + # refused `measure_not_offered` even though the read door offers the kind. + # ⛔ THE CELLS ARE DELIBERATELY ABSENT HERE. This ctx validates a write and + # materialises no rows at all (W30-T30 removed a full table build from the write + # path); computing measure values would put that work straight back. `measure_sets` + # stays empty for the same reason — a write is not a render. + # ⭐ AND THE ASYMMETRY IS SAFE, WHICH IS NOT OBVIOUS AND WAS CHECKED RATHER THAN + # ASSUMED. `routes_grid._ctx` derives TWO things from this dict and they have + # different jobs: `measure_keys` (from `measures`) is the ADMISSION list + # `clean_filter_tree` uses — non-empty here, so a saved measure CONDITION survives the + # write, which is the failure that comment records for the customer path. Empty + # `resolved_ids` (from `measure_sets`) only makes `view_upsert` answer "rerender", + # so the next `/workspace` — which DOES compute the sets, in `ut_assembly` — resolves + # it. Conservative in the right direction: one extra repaint, never a dropped filter. + # ⭐⭐ W38-T19 — THE CAPABILITY IS APPLIED HERE TOO, and this is the site that makes + # the refusal REAL rather than cosmetic: `routes_grid` reads this list as + # `measure_offer`, and `clean_measure_field` fail-closes `measure_not_offered` over + # an empty one. A picker hidden on the read door with this list still full would be + # a control that only stops the honest. + "measures": ut_measures(table_key, session), + "measure_sets": {}, "today": time.strftime("%Y-%m-%d"), + # ⭐ W31-T20 — the write door reads this to refuse a PID-BEARING event loudly rather + # than letting `allowed_pids` swallow it as a no-op. See `routes_grid`'s ut_ branch. + "limits": limits, "defn": defn} + + +def _ut_hidden(session, table_key, fields, st=None): + """The hidden-field closure for THIS session on THIS database — C1's field half, once. + + ⚠ Named rather than inlined at its four call sites for the reason `may_open`'s own note gives: + four spellings of one wall is how two of them come apart. `perm_scope.hidden_keys` is the ONE + evaluator; this is just the `ut_*` caller's shorthand for it. + + ⭐ W38-T16 — `st` IS THE LEND THE CALLER ALREADY HOLDS, and passing it is free rather than + merely tidy: the closure's field-grant leg resolves against `object_shares`, which is exactly + the bucket `ut_assembly`'s own lend note says the pass is already serving. + """ + import core.perm_scope as perm_scope + return perm_scope.hidden_keys(session.user, table_key, fields, st=st) + + +def _ut_shared_fields(session, table_key, fields): + """`fields` PLUS this database's TENANT-WIDE columns — the read-side merge, W38-T16. + + ⛔⛔ WHY THIS DID NOT EXIST AND WHY THAT WAS A HOLE. `patch_shared_cell` has written into + `__shared` for any `ut_*` database since W30-T28, and only the READ-THROUGH door + (`routes_odoo_tables`) ever merged that stratum back. So a shared column written on a + MATERIALISED user table went into the store and was readable by nobody, through any door — + complete, correct, and invisible ([[reachable-is-not-the-same-as-built]]). This is the other + half of that door. + + ⚠ MERGED BEFORE THE WALL, NEVER AFTER, and `routes_odoo_tables` learned this the same way: + the hidden closure must run on the WHOLE contract, or a formula in a shared column that reads + a hidden one sits outside its reach and carries the hidden value out wearing a second name. + ⚠ A key the definition already declares WINS. A shared column is an addition to a database's + contract, never a redefinition of a column that database already has. + """ + from core import shared_overlay + defs = shared_overlay.fields(table_key, st=session.runtime) or {} + if not defs: + return fields + have = {f.get("key") for f in (fields or ()) if isinstance(f, dict)} + return list(fields or ()) + [dict(f, source="overlay") + for k, f in defs.items() if k not in have] + + +def _ut_shared_cells(session, table_key, pids): + """`{"": {key: value}}` for the TENANT-WIDE stratum, over THIS session's row set. + + ⛔ `pids` IS THE ROW WALL AND IT IS PASSED, NOT DEFAULTED — `shared_overlay.cells` refuses an + "everything" read by signature for exactly this reason, and the set handed in is the one + `scoped_pool`/`scoped_pids` already narrowed. A cell for a row this session may not see + therefore has nothing to attach to. + """ + from core import shared_overlay + try: + return shared_overlay.cells(table_key, list(pids or ()), st=session.runtime) + except Exception: # noqa: BLE001 + return {} + + +def _ut_field_wall(session, table_key, fields, rows_src, st=None): + """`(fields, rows_src, hidden)` with the hidden closure removed from BOTH wires. + + ⭐⭐ W36-T21 — the same three lines `routes_customers.grid_assembly` runs for `customer_data`, + in the same position: AFTER `workspace_wire`, because the closure must cover the user's own + `custom_` and `measure_` columns and those do not exist until it has run. + + ⛔ BOTH WIRES, ALWAYS. `strip_row`'s docstring is the record of why: the field LIST and the + ROW payload are two different wires, and narrowing only the first leaves the value sitting in + the second where anything can read it. A formula (or a rollup) over a hidden column comes out + too — hiding the input while shipping the dependent either leaks the input wearing a derived + column's name or computes a wrong one. + + ⚠ IT TAKES NO `hidden` ARGUMENT, deliberately. The base-level closure the write ctx computed + is a SUBSET of this one by construction — same evaluator, a strictly larger field list — so + accepting it would be a second input that can only ever be redundant, i.e. a parallel code + path with nothing to say ([[one-question-two-normalizers]]). + """ + import core.perm_scope as perm_scope + hide = perm_scope.hidden_keys(session.user, table_key, fields, st=st) + if not hide: + return fields, rows_src, frozenset() + fields = [f for f in fields if f.get("key") not in hide] + rows_src = [perm_scope.strip_row(r, hide) for r in (rows_src or [])] + return fields, rows_src, hide + + +def ut_assembly(session: Session, table_key: str, storage_key: str = "", + consume_corrections: bool = True, with_rows: bool = True, st=None): + """The user-table mirror of `grid_assembly` / `product_assembly` — SAME g-dict keys, so + `/workspace` and the events route consume any of the three interchangeably. + + Honest absence: `measures`/`measure_sets` are EMPTY — `core.measure_resolve` is + customer-grain, so there is nothing to offer over user rows. + + ⭐ WAVE 19 / R9 — `lists` IS NO LONGER EMPTY. "For ANY database new/old": a user table gets + cohorts like every other database, out of its OWN bucket (`ut__cohorts`), holding its + own row ids. The wave-18 refusal was correct while there was one customer-keyed bucket and + wrong the moment the store learned about topics. + + ⭐⭐ W31-T20 / D-174 — `with_rows=False` BUILDS THE ENVELOPE AND NOT THE TABLE, and the + caller that wants it is `/workspace`, which renders no row at all (the grid fetches rows from + `/tables/{key}/rows` or `/odoo-tables/{key}/rows` beside it). Two things follow: + * the read-through line grains become OPENABLE — `scoped_pool` refused their envelope over + 963,783 rows nobody was going to look at, which is D-174 in one sentence; + * every materialised `ut_*` database stops allocating a dict per row and sorting them on a + route whose payload has no rows in it — `ut_odoo_orders` was rebuilding 32,826 of them per + database switch (owner item 7). + ⛔ NOTHING IS VALIDATED LESS. `scoped_pids` runs the SAME `_defn_or_refuse` wall and answers + the identical pid set; the flag removes work, never a check — the shape W30-T30 already proved + on the write door. + """ + import aios_grid + from core import grid_events + + limits = [] + # ⭐⭐ W36-T24 / D-214 — **ONE LEND FOR THE WHOLE PASS**, and it is the shape of the lend that + # keeps D-213's saving. The rows arm needs `rows` off the definition, so it lends the whole + # document; the envelope arm reads no row at all and lends the PROJECTION. Either way the same + # object then serves `object_shares` to the grant legs downstream, so the assembly stops + # reading that bucket once per question asked of it. + if st is None: + st = _ut().lend(session.runtime) if with_rows else _ut().lend_defs(session.runtime) + if with_rows: + # ⭐ W33-T03 (D-214): `st` is a caller's LEND, threaded to the wall and nowhere else. Only + # a READ route passes one — see `scoped_pool`'s own note and contract C5. + pids, rows_src, fields_base, defn = scoped_pool(session, table_key, st=st) + else: + pids, fields_base, defn = scoped_pids(session, table_key, limits=limits, st=st) + rows_src = [] + + # ⭐ W36-T24 / D-214 — ONE lend for the whole pass. A caller that already holds one passes it + # as `st`; otherwise this assembly takes its own. Only `user_tables` and `object_shares` are + # served from it, and the assembly writes neither. + ops = _ops(session, table_key, st=st) + # ⭐⭐ W36-T21 — the WRITE half of the field wall. `frozenset()` here meant a column an + # administrator had hidden was still writable through the events transport. + hidden = _ut_hidden(session, table_key, fields_base, st=st) + ctx = grid_events.EventCtx( + uname=session.uname, allowed_pids=pids, fields=[], hidden_keys=hidden, + admin=session.admin, fallback_ws=None, seen_ids={}, + # R6b (D-16): the tenant handle rides every ctx this layer builds, not just the ones + # that happen to carry a scoped `table`. + st=session.runtime, + scope_key=table_key, table=ops) + ws = grid_events.table_workspace(ctx, allowed_pids=pids, + consume_corrections=consume_corrections) + workspace, fields, views, lists = aios_grid.workspace_wire( + ws, session.uname, set(pids), defs={}, scope_key=table_key, + storage_key=storage_key, fields_base=fields_base) + # ⭐⭐ W38-T16 — THE TENANT-WIDE STRATUM, ON A MATERIALISED GRID. Merged HERE, between + # `workspace_wire` and the wall, which is `routes_odoo_tables`' own position for the same two + # lines and for the same reason: the closure must be recomputed on the MERGED contract or a + # formula in a shared column reaches past it. + fields = _ut_shared_fields(session, table_key, fields) + # ⚠ THE CELLS COST A READ, SO THE ENVELOPE ARM DOES NOT PAY IT. `with_rows=False` renders no + # row at all (D-174), and `cells()` over the whole pid set would be a scoped read whose + # result nothing consumes. The COLUMNS still merge above: `/workspace` needs the contract. + shared_cells = _ut_shared_cells(session, table_key, pids) if with_rows else {} + # ⭐⭐ W36-T21 / R6 — the READ half, in `grid_assembly`'s own position: after `workspace_wire`, + # so the closure covers this user's `custom_` and `measure_` columns too. + fields, rows_src, hidden = _ut_field_wall(session, table_key, fields, rows_src, st=st) + if hidden and shared_cells: + # ⛔ BOTH WIRES, AND THE SHARED STRATUM IS A THIRD ONE. `strip_row` above cleaned the + # DEFINITION rows; these cells arrived by a different door and one narrowing cannot speak + # for both — the same sentence `routes_odoo_tables` writes over its own `overlays` dict. + shared_cells = {pid: {k: v for k, v in cells.items() if k not in hidden} + for pid, cells in shared_cells.items()} + + # ⭐⭐ W37-T12 / owner item 4 (R1) — `measures` IS NO LONGER UNCONDITIONALLY EMPTY. A database + # whose entity topic declares a `measures:` binding (today `ut_odoo_agents`) serves a real + # lookback catalogue AND the cells to go with it; every other user table still serves `[]`, + # which is the honest answer for a hand-typed one rather than a descope. + today = time.strftime("%Y-%m-%d") + measures = ut_measures(table_key, session) + derived = aios_grid.cohort_cells(lists) # R9: this table's own lists + for _pid, _cells in _ut_measure_cells(table_key, fields, pids, today, measures, + session.runtime.measure_memo).items(): + derived.setdefault(_pid, {}).update(_cells) + + return {"rows_src": rows_src, "pids": pids, "ws": ws, "workspace": workspace, + "fields": fields, "views": views, "lists": lists, "hidden": hidden, + "derived": derived, + # ⭐ W38-T16 — ALWAYS PRESENT, `{}` when this database shares nothing. A key a + # consumer has to test for is a key a consumer forgets to test for, and the consumer + # here (`table_rows`) is layering strata in a fixed order. + "shared_cells": shared_cells, + "measures": measures, + "measure_sets": _ut_measure_sets(table_key, views, measures, pids, today), + "today": today, + # ⚠ ALWAYS PRESENT, EMPTY WHEN THERE IS NOTHING TO SAY — a key a consumer has to test + # for is a key a consumer forgets to test for, and this one carries a refusal. + "limits": limits, "defn": defn} + + +def _ut_measure_err(tag, e): + try: + import harness.telemetry as _tel + _tel.error(f"api:{tag}", e) + except Exception: # noqa: BLE001 + pass + + +def ut_measures(table_key, session): + """The lookback-measure OFFER for a user database FOR THIS SESSION, or `[]` — W37-T12 / + owner item 4 (R1), narrowed by the metrics capability in W38-T19. + + ⭐⭐ W38-T19 — `session` IS REQUIRED, NOT DEFAULTED. The function was PRINCIPAL-BLIND, so a + per-user capability had nothing to be consulted by. Defaulting it would leave both existing + call sites compiling and silently ungated, which is the fail-open spelling of the bug rather + than a fix for it — the argument `_module_fields` in `routes_admin` records for its own + `session`. ⛔ AND BOTH CALL SITES MATTER, WHICH IS EASY TO GET HALF RIGHT: `ut_assembly` is + the READ and `ut_write_ctx` is the door `routes_grid` turns into `clean_measure_field`'s + admission set. Gating only the read would take the kind off the picker and leave the CREATE + working for anyone who posts the event. + + ⭐ THE SAME ENGINE THE PRODUCT GRID USES (`semantic.entity_measures`), reached through the + topic that DESCRIBES this database. A table with no entity topic, or a topic with no + `measures:` binding, gets `[]` — which is the honest answer for a hand-typed database and is + what keeps the Metric kind hidden there (`ColumnMenu`'s `offerMeasure={measures.length > 0}`). + + ⛔ NOT A `{table_key: [keys]}` MAP IN THIS FILE. The binding is declared in the model + (`model/topics/odoo_agents.yml`'s `measures:` block) and the topic already names its own grid, + so a list here would be a second definition of one fact — the argument `_rollup_source_offer` + makes about itself one screen down. + """ + import core.perm_scope as perm_scope + if not perm_scope.may_metrics(session.user, table_key): + return [] + try: + from harness import semantic as sem + topic = sem.topic_for_grid(table_key) + return sem.entity_measures(topic) if topic else [] + except Exception as e: # noqa: BLE001 + _ut_measure_err("ut-measure-offer", e) + return [] + + +def _ut_measure_cells(table_key, fields, pids, today, offer, memo, team_id=None): + """`{pid: {field key: value}}` for this database's Metric columns. + + ⭐ THE JOIN NEEDS NO MAP HERE, and that is worth stating because the product grid DOES need + one. `sales_lines.agent` is `rp.agent_id` — a `res.partner` id — and a `ut_odoo_agents` row's + `pid` IS its `res.partner` id (`scoped_pool` sets `pid = int(rid)` and `read_agents` keys + `_id` on the partner id). So the group key and the pid are the same integer. The product grid + keys on `default_code` instead, which is exactly why contract C1 calls the join key a trap: + the right answer differs per database and must be read off the binding, never assumed. + """ + import aios_grid + + mfields = aios_grid.measure_fields_of(fields) + if not mfields or not offer: + return {} + from harness import semantic as sem + from harness import windows as _wn + + topic = sem.topic_for_grid(table_key) + by_key = {m["key"]: m for m in offer} + by_window = {} + for f in mfields: + spec = f.get("measure") or {} + if spec.get("key") not in by_key: + continue + rng = _wn.resolve(spec.get("window"), today) if _wn.normalize(spec.get("window")) else None + if rng is None: + continue # an unresolvable window is unanswerable — never widen to all time + by_window.setdefault((rng[0], rng[1]), []).append(f) + + out = {} + for (dfrom, dto), group in by_window.items(): + keys = tuple(sorted({f["measure"]["key"] for f in group})) + mkey = ("entity", topic, table_key, today, team_id, dfrom, dto, keys) + if mkey not in memo: + try: + memo[mkey] = sem.entity_measure_values(topic, list(keys), date_from=dfrom, + date_to=dto, team_id=team_id, offer=offer) + except Exception as e: # noqa: BLE001 + _ut_measure_err("ut-measure-column", e) + transient = False + try: + from harness import datastore as _ds + transient = not _ds.ready() + except Exception: # noqa: BLE001 + transient = False + if transient: + continue # a warming store must not memoise as a permanent failure + memo[mkey] = None + vals = memo[mkey] + if vals is None: + continue + # C1's empty-window rule for EVERY row, not only the grouped ones. + filled = {pid: sem.entity_zero_fill(vals.get(pid), list(keys), offer) for pid in pids} + for f in group: + k, fkey = f["measure"]["key"], f["key"] + for pid, cell in filled.items(): + if k in cell: + out.setdefault(pid, {})[fkey] = cell[k] + if len(memo) > 100: + memo.clear() + return out + + +def _ut_measure_sets(table_key, views, offer, pids, today, team_id=None): + """`{rule id: [pid, …]}` for the measure CONDITIONS in this database's saved views. + + ⛔ SHIPS WITH THE OFFER, NEVER AFTER IT. `routes_grid` derives `measure_keys` from the same + list, so a non-empty `measures` also opens the measure condition in the filter builder — and a + rule id missing from `measureSets` renders PENDING ("Calculating…") and matches nothing, + permanently. Offer and sets are one feature. + """ + if not offer: + return {} + from harness import semantic as sem + + topic = sem.topic_for_grid(table_key) + admitted = {m["key"] for m in offer} + by_id = {} + for v in views or []: + for rule in sem.entity_measure_leaves((v.get("config") or {}).get("filters"), admitted): + rid = str(rule.get("id") or "") + if rid: + by_id[rid] = rule + if not by_id: + return {} + try: + sets = sem.entity_measure_sets(topic, list(by_id.values()), today, team_id=team_id, + keys_by_id=list(pids), offer=offer) + except Exception as e: # noqa: BLE001 + _ut_measure_err("ut-measure-sets", e) + return {} + return {rid: sorted(int(k) for k in ks if str(k).lstrip("-").isdigit()) + for rid, ks in sets.items()} + + +def ut_label(defn, key, meta=None): + """THE name of a user table, resolved ONCE (wave 20, item 6a). + + `nav_meta`'s rename wins, then the definition's own label, then the key. Every surface that + shows a database name reads through here, because the alternative is what wave 20 found: the + rail showed the renamed name (it reads `nav_meta`) while the automation editor's picker + showed the original (it reads the definition), and neither looked broken. + + ⚠ `set_label` now writes the DEFINITION too, so the two agree at the source. This resolver + stays because it makes every row already stored — renamed before that fix landed — read + correctly today, without a migration. + """ + return ((meta or {}).get(key, {}).get("name") + or (defn or {}).get("label") or key) + + +def nav_meta(session): + """The tenant's nav_meta bucket, read defensively. A store blip must not take a list down.""" + try: + got = session.runtime.get("nav_meta") + return got if isinstance(got, dict) else {} + except Exception: # noqa: BLE001 + return {} + + +@router.get("/tables") +def list_tables(session: Session = Depends(require_session)): + """This session's user tables — the list the '+ New database' surface renders. + + ⭐⭐ W31-T12 (contract C1, D-175's third instance) — ONE DOCUMENT READ, NOT `1 + 2N`. This + route was never ticketed and has the same shape `/nav` and `/automations` were fixed for: + `all_tables` once, then `may_open` per key (another whole-document deep copy each, 28.6 MB on + tenant #0 measured, 703 ms warm) and `records_mutable` per key on top of that — so a tenant + with ten databases paid twenty-one copies to list them. The wall is UNCHANGED and still asked + about every table; it is handed the document this function already holds. See + `user_tables.lend`'s own note for why inlining the predicate is the one fix that is not + available. + """ + ut = _ut() + meta = nav_meta(session) + out = [] + tables = ut.all_tables(st=session.runtime) + lent = ut.lend(session.runtime, **{ut.STORE_KEY: tables}) + for key, t in sorted(tables.items(), + key=lambda kv: (ut_label(kv[1], kv[0], meta) or "").lower()): + if not ut.may_open(key, session.uname, session.admin, st=lent): + continue + out.append({"key": key, "label": ut_label(t, key, meta), + "source": t.get("source") or "Blank", + "recordsMutable": ut.records_mutable(key, st=lent), + "createdBy": t.get("createdBy") or "", + "created": t.get("created") or "", + "fields": [dict(f) for f in (t.get("fields") or [])], + "rowCount": len(t.get("rows") or {})}) + return {"tables": out} + + +#: ⭐⭐ THE ROLLUP SOURCE OFFER — what makes the read-through rollup a FEATURE rather than a +#: capability. Owner 2026-08-09: *"Full field editor: pick topic → metric → window."* +#: +#: ⛔ THE ENGINE SHIPPED WITH NO WRITER. `_clean_rollup` has accepted a `source` bag since +#: 2026-08-09 and `rollup_sql` answers it in one grouped query, but the ONLY thing in the product +#: that ever produced one was a hard-coded field in `odoo_relational.py`. Nothing in the client +#: could make one, so the whole read-through path was reachable by editing Python — the +#: [[artifact-with-no-importer]] shape, twice burned in this repo already. +#: +#: ⚠ DERIVED FROM THE MODEL FILES, NEVER A HAND-WRITTEN LIST. `model/topics/*.yml` and +#: `model/metrics/*.yml` already state which measures belong to which topic and which dims that +#: topic can group by; a second list here would be a second definition of the same fact, and the +#: two would drift the day somebody adds a metric. Same argument `rollup_sql` makes for binding +#: to a metric KEY instead of carrying SQL. +_ROLLUP_CACHE = {} + + +def _rollup_source_offer(): + """`{topics:[…], windows:[…]}` — every (topic, measure, dim) the engine can actually answer. + + ⛔ ONLY COMBINATIONS THAT CAN RESOLVE ARE OFFERED, because a rollup that refuses at COMPUTE + time refuses silently — the cells are simply left blank, hours later, on a column that looks + configured. Two exclusions do real work: + * a topic with NO dims cannot be grouped at all, so it can never key a parent row; + * a CROSS-TOPIC metric (`aov`, `margin_pct`, `returns_pct` — `agg: ratio`/`derived` whose + inputs live elsewhere) is refused by `store_query` the moment a `group_by` is present: + *"cross-topic measures are scalar-only"*. Offering one would mint a column that can only + ever error. + Each dim also declares HOW it keys — by an Odoo id or by its own value — because that is what + the user is matching their own column against, and `payment_state` (a value) and + `partner` (an id) are matched to very different columns. + """ + if _ROLLUP_CACHE.get("offer"): + return _ROLLUP_CACHE["offer"] + from harness import semantic as sem + from harness import windows as W + ut = _ut() + + topics, metrics = sem.topics(), sem.metrics() + by_topic = {} + for key, m in metrics.items(): + # A ratio/derived metric whose parts sit on another topic cannot be grouped — see above. + if m.get("agg") in ("ratio", "derived"): + continue + by_topic.setdefault(m["topic"], []).append( + {"key": key, "label": m.get("label") or key, "format": m.get("format") or "usd", + "description": m.get("description") or ""}) + + out = [] + indeterminate = False + for tkey, t in sorted(topics.items()): + dims = ((t.get("store") or {}).get("dims") or {}) + measures = by_topic.get(tkey) or [] + if not dims or not measures: + continue + # ⛔⛔ W37-T13 / /validate-wave 2026-08-19 — THE SOURCE TABLE MUST BE LIVE, AND THIS DOOR + # DID NOT ASK. The docstring above already promises *"ONLY COMBINATIONS THAT CAN RESOLVE + # ARE OFFERED"*; adding `stock_move` to `datastore.ENTITIES` broke that promise on the one + # door nobody re-read. `semantic._one_binding_offer` — the MEASURE-CATALOG door — refuses + # the same bindings with a stated cause, so the wave shipped **two doors onto one + # vocabulary with only one of them guarded**, and the unguarded one is the picker. + # MEASURED: `_rollup_source_offer()` listed `stock_moves` with both measures and all four + # dims while `royal.duckdb` has no `stock_move` table at all, so choosing it raised + # `CatalogException: Table with name stock_move does not exist!` — five of the fifty + # (topic, measure, dim) triples in `verify_odoo_relational`. A user meets a raw catalog + # error, or a column that looks configured and is blank forever + # ([[permitted-is-not-answerable]]). + # ⚠ REUSES `_source_ready` rather than re-deriving readiness: one question, one + # normalizer. `None` (the store could not be asked) refuses TOO — fail closed, exactly as + # the measure door does. + _tbl = (t.get("store") or {}).get("table") + if _tbl: + _rdy = sem._source_ready(_tbl) + if _rdy is not True: + # ⛔ AND AN INDETERMINATE ANSWER MUST NOT BE CACHED. `_source_ready`'s own comment + # records the measured version of this: a cold container with no mirror yet + # refused every binding, the refusal was memoised, and the grid served no measures + # long after the mirror arrived. Here the same shape would freeze a truncated + # PICKER for the life of the process. + indeterminate = indeterminate or (_rdy is None) + continue + out.append({ + "key": tkey, + "label": t.get("label") or tkey, + "grain": t.get("grain") or "", + "dims": [{"key": dkey, + "label": d.get("label") or dkey, + # `store_query` emits `_id` only when the dim carries a display name + # alongside the key; otherwise the value IS the key. `rollup_sql` handles + # both, and the editor says which so the user matches the right column. + "keyedBy": "id" if d.get("name_col") else "value"} + for dkey, d in dims.items()], + "measures": sorted(measures, key=lambda m: m["label"].lower()), + }) + + # ⚠ THE WINDOW LIST IS `core.user_tables`' OWN, not `harness.windows`'. `ROLLUP_SOURCE_WINDOWS` + # is the validator's closed set and is deliberately NARROWER (it omits the parameterised kinds + # like `last_n_days`, which have nowhere in the bag to carry their `n`). Offering a kind the + # validator refuses would let the editor build a field the save door rejects. + windows = [{"key": k, "label": W.WINDOW_LABELS.get(k, k).format(n="N")} + for k in ut.ROLLUP_SOURCE_WINDOWS] + offer = {"topics": out, "windows": windows} + # ⚠ Cache only a DEFINITE answer (see the readiness block above): if any topic was dropped + # because the store could not be asked, this offer is a snapshot of an unknown, not a fact. + if not indeterminate: + _ROLLUP_CACHE["offer"] = offer + return offer + + +@router.get("/tables/rollup-sources") +def rollup_sources(session: Session = Depends(require_session)): + """The topic → metric → dim → window offer the rollup field editor renders. + + ⚠ DECLARED ABOVE EVERY `/tables/{table_key}/…` ROUTE, and kept there deliberately. FastAPI + matches in declaration order, so the day somebody adds a bare `GET /tables/{table_key}` below + this line it still resolves; added ABOVE it, this endpoint would silently start arriving as + `table_key='rollup-sources'` and 404 from the table wall. There is no such route today — + this is the cheap ordering that keeps it from mattering. + + ⛔ TENANT-SCOPED, AND IT WAS NOT WHEN FIRST WRITTEN. `sem.topics()` reads the GLOBAL model + files, so nurilab and gtmlab were served the full Odoo offer — they would have seen "Live + Odoo data" in the field editor and been able to build a column that can only ever be blank, + because there is no mirror behind it. Three comments (here, in `apiBridge` and on the + `rollupSourceOffer` prop) each asserted that a tenant with nothing connected receives `[]`, + and the mode switch's "render only when there is a choice" guard is built on that promise. + ⚠ `odoo_relational.is_royal` is the authority, reused rather than re-decided: it is already + what `refresh` consults to decide whether these tables may exist at all, and a second copy of + the rule would be a second answer the day a tenant gains a mirror. + """ + import odoo_relational + if not odoo_relational.is_royal(session.tenant): + return {"topics": [], "windows": []} + return _rollup_source_offer() + + +@router.post("/tables", status_code=201) +def create_table(body: dict = Body(default=None), + session: Session = Depends(require_session)): + ut = _ut() + body = body or {} + label = str(body.get("label") or "").strip() + if not label: + raise err(400, "bad_label", "give the database a name") + if not session.runtime.available(): + raise err(503, "store_unavailable", + "the tenant store is unavailable. Nothing was created") + source = body.get("source") + try: + key = ut.create(label, session.uname, fields=body.get("fields"), + source=source, st=session.runtime) + except Exception: + raise err(503, "store_unavailable", + "the tenant store refused the write. Nothing was created") + if not key: + raise err(400, "refused", + f"could not create it. The name may be empty or this tenant already has " + f"{ut.MAX_TABLES} databases") + return {"key": key} + + +@router.patch("/tables/{table_key}") +def patch_table(table_key: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Rename a database — IN ITS DEFINITION (wave 20, item 6a). + + ⚠ THE RENAME DOOR IN THE NAV WRITES `nav_meta` AND MUST ALSO CALL THIS. `nav_meta` is the + nav's display layer; the definition is what the automation editor's database picker, the + schema drawer and every future reader see. A rename that lands in only one of them leaves a + picker that is confidently wrong rather than obviously stale. Posted to the wave doc as an + amendment for whoever owns that door. + """ + defn = _defn_or_refuse(session, table_key) + ut = _ut() + if not (session.admin or defn.get("createdBy") == session.uname): + raise err(403, "forbidden", "only the database's creator or an admin can rename it") + label = ut.set_label(table_key, (body or {}).get("label"), st=session.runtime) + if not label: + raise err(400, "bad_label", "give the database a name") + return {"key": table_key, "label": label} + + +@router.get("/tables/{table_key}/footprint") +def table_footprint(table_key: str, session: Session = Depends(require_session)): + """What dies with this database — the confirm dialog's disclosure (wave 21, item 6a / C3). + + Counts drill to the SAME buckets `user_tables.delete` cleans; a dialog listing categories + without numbers would break [[no-unverifiable-aggregates]] at the scariest moment. Walled + like the delete itself: only someone who could delete may case the joint. + + ⭐ W33-T01 / D-213: the wall and `createdBy`/`fields` come off the PROJECTION; only the row + COUNT needs the whole document, and only on a materialised table.""" + defn = _defn_or_refuse(session, table_key, defs_only=True) + if not (session.admin or defn.get("createdBy") == session.uname): + raise err(403, "forbidden", "only the database's creator or an admin can delete it") + s = session.runtime + views, fields = set(), len(defn.get("fields") or []) + try: + bucket = s.get(f"{table_key}_table_workspace") or {} + for _u, ws in bucket.items(): + if isinstance(ws, dict): + views |= set((ws.get("views") or {}).keys()) + fields += len(ws.get("fields") or {}) # per-user custom/measure strata + except Exception: + pass + import core.shares as shares + g = shares.grants("database", table_key, st=s) + auto = [] + try: + import automation_engine as engine + for aid, d in (engine.all_definitions(s) or {}).items(): + if (d.get("config") or {}).get("targetTable") == str(table_key): + auto.append({"id": str(aid), "name": d.get("name") or str(aid)}) + except Exception: + pass + # ⛔ THE ROW COUNT IS THE ONE FIELD THAT NEEDS THE WHOLE DOCUMENT, and it needs it only where + # the rows are actually stored here. A read-through database keeps `rows: {}` by construction, + # so `len(...)` answered **0** for it before this change and answers 0 now — identical, and the + # projection is not what makes it wrong. + # ⚠ 0 IS A WRONG NUMBER FOR A READ-THROUGH GRID and always was (`ut_odoo_gl_lines` would say 0 + # in a dialog headed "what dies with this database"). Booked rather than fixed here: this + # ticket is a read-path change and correcting it means asking the mirror for a `count(*)` + # inside a confirm dialog. See the `PENDING:` line in `mailbox/A.md`. + if _ut().materialises(table_key, st=s, defn=defn): + rows = len((_ut().get(table_key, st=s) or {}).get("rows") or {}) + else: + rows = 0 + return {"rows": rows, "fields": fields, "views": len(views), + "sharedUsers": len(g.get("entries") or []), + "automations": sorted(auto, key=lambda a: a["name"].lower())} + + +@router.delete("/tables/{table_key}") +def delete_table(table_key: str, session: Session = Depends(require_session)): + """CREATOR OR ADMIN — checked explicitly (wave 21, item 6a / C3). + + ⛔ The wave-20 docstring said "the same actors may_open admits" and that stopped being the + creator-or-admin set the day de5037f taught `may_open` to admit share GRANTEES: a view-role + grantee could reach this route and delete the database somebody shared with them. The wall + is now the definition's own `createdBy`, the same check the rename route always had. + + Deletion cleans the artifact families server-side (`user_tables.delete` lists them) and + DISABLES bound automations with a status note — never deletes them. The client's confirm + dialog disclosed `/footprint` first; the server cannot tell a click from a plan, so the + dialog is a product requirement, not a formality.""" + defn = _defn_or_refuse(session, table_key) + if not (session.admin or defn.get("createdBy") == session.uname): + raise err(403, "forbidden", "only the database's creator or an admin can delete it") + try: + import automation_engine as engine + engine.disable_for_table(session.runtime, table_key) + except Exception: + pass + try: + _ut().delete(table_key, st=session.runtime) + except Exception: + raise err(503, "store_unavailable", "the delete did not land. Try again") + return {"ok": True} + + +#: ⭐ 2026-08-07 — tenants whose Instagram tables THIS PROCESS has already brought forward. +_IG_FORWARDED = set() + + +def _ig_forward(session): + """Bring this tenant's Instagram tables onto the current schema, at most once per process. + + ⛔ WHY A MIGRATION RUNS ON A READ AT ALL, when the module's own rule is that it rides the WRITE + path. `ut_ensure` calling it is right for a schema an automation is about to append to, and + useless for a change a PERSON is waiting to see: the owner's report was *"the first field is + still blank"*, and "re-save the automation and it will fix itself" is not an answer to that. + The write path stays exactly as it was — this is a second door to the same idempotent call, + not a replacement for it. + + ⚠ BOUNDED THREE WAYS, because a write on a read is otherwise how a grid gets slow: once per + tenant per process; `migrate_ig_tables` returns without a write when every table is already + current (the common case after the first read); and a failure is SWALLOWED — a migration must + never be the reason a database will not open. + + ⚠ THE TENANT IS MARKED BEFORE THE ATTEMPT, deliberately. A migration that raises must not be + retried on every subsequent read of every table for the life of the process — the write path is + still the backstop, so the cost of skipping is a delay, while the cost of retrying is a failing + store call on the hot path of a grid that is trying to render. + """ + tenant = str(getattr(session, "tenant", "") or "") + if tenant in _IG_FORWARDED: + return + _IG_FORWARDED.add(tenant) + try: + import automation_engine as engine + engine.migrate_ig_tables(session.runtime, log=lambda *_a: None) + except Exception as e: # noqa: BLE001 + print(f"[tables] ig forward-migration skipped: {type(e).__name__}: {e}") + + + +#: Above this many characters a `json` cell is replaced by a stand-in in the LIST envelope. Sized +#: so an ordinary config document (a few hundred bytes) is untouched while a vendor response is +#: not — the shape this exists for is one already-paid provider payload per row. +JSON_LIST_MAX = 400 + + +def _thin_json(fields, merged, table_key=""): + """Replace oversized `json` cells with a stand-in for the LIST response. Pure; returns a copy. + + ⚠ THE STAND-IN IS ITSELF VALID JSON and carries the byte count, so the grid preview reads + `{...} 3 keys` rather than a broken brace, and a reader can see the column holds something + large rather than something empty. `_truncated` is what the viewer keys its fetch on. + + ⭐ THE STAND-IN CARRIES ITS OWN `_url`, which is what keeps this change small AND correct: the + viewer needs no table key, no record id and no new props threaded down through three + components to find the document — the route that removed the value says where it went. One + writer of that address instead of a server rule and a client rule that must agree forever. + """ + json_keys = [str(f.get("key")) for f in (fields or []) + if str(f.get("type") or "") == "json"] + if not json_keys: + return merged + out = {} + for pid, cells in (merged or {}).items(): + row = cells + for key in json_keys: + raw = cells.get(key) + if isinstance(raw, str) and len(raw) > JSON_LIST_MAX: + if row is cells: + row = dict(cells) + row[key] = json.dumps({ + "_truncated": True, "bytes": len(raw), + "_url": f"/api/v1/tables/{table_key}/rows/{pid}/fields/{key}"}) + out[pid] = row + return out + + +@router.get("/tables/{table_key}/rows") +def table_rows(table_key: str, session: Session = Depends(require_session)): + """The `/customers`-shaped envelope for one user table: `{fields, rows, today, pulled_at, + identity}` — so the client's generic topic fetch consumes it with zero new parsing. + + ⚠ THE MERGE ORDER IS THE CONTRACT. `rows_from_pool` sources an overlay-typed field's cell + from the OVERLAY stratum only (that is what makes a custom column render standalone) — a + user table's base values live in its DEFINITION rows, so they are layered UNDER the user's + overlay edits here: base first, overlay wins. Without this every base cell reads empty + (found by this route's own gate check, not by luck).""" + import aios_grid + + _ig_forward(session) + # ⭐⭐ W33-T03 / D-214 — ONE READ OF THE TENANT DOCUMENT FOR THE WHOLE REQUEST, MEASURED. + # This route asked for it FOUR times: the wall (via `scoped_pool`), then `limit_report` TWICE + # (`row_limit` calls `materialises` and then `is_connected`, and each takes its own whole copy), + # then `records_mutable` at the envelope. Each is ~703 ms warm on tenant #0, and all four ask + # about the SAME document in the SAME request. The lend is the fix W31 QA already built for + # `_defn_or_refuse`; this threads it through the three sites that never got it. + # ⚠ SAFE HERE FOR THE SAME REASON IT IS SAFE THERE: this is a pure READ route. A lend is a + # PRE-WRITE snapshot, and handing one to a path that writes and then reads back would report the + # value it replaced (contract C5). + _lent = _ut().lend(session.runtime) + g = ut_assembly(session, table_key, st=_lent) + # ⛔ THE OVERLAY WAS NEVER ACTUALLY MERGED, and the docstring above has described the merge + # this line does not perform since the route was written (owner item 3, 2026-08-09: + # *"Using the swipe, fast doesn't register the CHANGE. I went back and it all got reseted"*). + # + # `merged` was built from `rows_src` ALONE — the DEFINITION rows. But `rows_from_pool` + # sources an overlay-typed field's cell from this dict, and a CUSTOM column on a `ut_*` table + # is overlay-typed by construction, so it looked up a key that could not be there and every + # such cell rendered blank. + # + # ⭐ THE WRITES WERE NEVER LOST — MEASURED. `ws['overlays']` holds + # `{"1": {"custom_geography_yf6vi": "Jakarta"}, ...}` for ten rows: the owner's swipes landed + # in the store exactly as they should. Only the READ-BACK dropped them, which is why the + # value survived the gesture, vanished on reload, and looked like "it reset itself" — and + # why `patch_row`'s `_took()` then reported a perfectly good write as `refused`. + # + # ⚠ OVERLAY WINS, base underneath — the order the docstring already specifies. A definition + # value must not shadow an edit the user has made on top of it. + # ⚠ AND IT IS THIS USER'S OWN OVERLAY (`table_workspace` is keyed by `ctx.uname`), so this + # widens what a caller can SEE by exactly their own edits and nothing else. + _overlays = (g.get("ws") or {}).get("overlays") or {} + merged = {} + for _r in g["rows_src"]: + _pid = str(_r["pid"]) + _cells = {k: v for k, v in _r.items() if k != "pid"} + _ov = _overlays.get(_pid) + if isinstance(_ov, dict): + _cells.update(_ov) + merged[_pid] = _cells + # ⭐⭐ W38-T16 — THE TENANT-WIDE STRATUM GOES ON TOP, and the order is the contract rather + # than a preference. A shared column is one the whole permitted audience must read the SAME + # value in — that is the entire reason `shared_overlay` exists — so a stale per-user value + # left under the same key must not win. `routes_odoo_tables` layers the two in this exact + # order (`overlays.setdefault(pid, {}).update(cells)`) and the two doors must agree. + # ⛔ NEVER `setdefault` ON `merged`: these cells are already narrowed to this session's pids, + # but inventing a row id here would put a row in the payload that `rows_src` never admitted. + for _pid, _cells in (g.get("shared_cells") or {}).items(): + if str(_pid) in merged and isinstance(_cells, dict): + merged[str(_pid)].update(_cells) + # ⭐⭐ 2026-08-10 (owner: *"wth is going on, why does it take forever to load now?"*) — THE + # JSON DOCUMENTS DO NOT RIDE THE LIST. + # + # MEASURED on nurilab, and the numbers are the whole argument: `source_payload` is **95.6% + # to 98.5%** of every IG grid's bytes — `ut_ig_post_snapshots` shipped **11.6 MB of a 12.2 MB + # response**, `ut_ig_snapshots` 1.85 MB of 2.03 MB, the profile table 1.83 MB of 2.11 MB. The + # grid renders those cells as a SIXTY-CHARACTER preview (`display.jsonPreview`), so the whole + # vendor response crossed the wire, was parsed by the browser and held in memory purely so a + # clipped first line could be drawn. + # + # ⚠ IT IS NOT A CAP AND NOTHING IS LOST. The full document is served by + # `GET /tables/{key}/rows/{pid}/fields/{fkey}`, which the JSON viewer fetches when it opens — + # the one place a person actually reads it, for the one row they opened. The cell that rides + # the list is a VALID small document saying what it stands for, so the preview renders + # honestly instead of showing half a truncated brace. + # ⛔ ONLY `json` COLUMNS, and only over the threshold: a small document still travels whole, + # so a tenant using `json` for a short config sees no change at all. + rows = aios_grid.rows_from_pool( + g["rows_src"], g["fields"], _thin_json(g["fields"], merged, table_key), derived=g["derived"]) + # ⭐⭐ WAVE-34 (R13) — the per-cell enrichment STATE rides the row, beside `_created`/`lat`/ + # `lon`. See `_stamp_ai_states` for why it is a row key rather than a map beside `rows`. + _stamp_ai_states(table_key, g["fields"], rows, st=_lent) + # ⭐ R6's SECOND SENTENCE, ON THE WIRE (W30-T29). *"if there is lag or it can't be done, you + # need to explicitly tell me why and recommend a fix."* A ceiling that still applies to this + # database says so here, with its cause and the recommendation, rather than waiting to be + # discovered as a refused paste. `None` for a connected source, and an EMPTY LIST is the + # honest answer for a table nothing limits — never an absent key, which a client cannot tell + # apart from an older server. + _report = _ut().limit_report(table_key, st=_lent) + # ⭐ C4 / D-138 — THE DOCUMENTS PRODUCER. Absent since EXIT-6 deleted `app.py`, which was the + # only thing that ever set this key; the write door never stopped working and every client + # half is complete, but all six `onDoc*` handlers read `payload?.docs ? … : undefined`, so a + # missing key has been silently switching the feature off. ⛔ ONE shared serialiser with the + # customer scope — `grid_events.docs_for` — never a twin here. + from core import grid_events as _ge + return {"fields": g["fields"], "rows": rows, "today": g["today"], + "docs": _ge.docs_for(g["pids"], scope_key=table_key, uname=session.uname, + admin=session.admin, st=session.runtime), + "pulled_at": time.strftime("%Y-%m-%d %H:%M"), + "identity": {"pid": "pid"}, + "scope": {"table": table_key, "rowCount": len(rows)}, + "limits": [_report] if _report else [], + "recordsMutable": _ut().records_mutable(table_key, st=_lent)} + + +#: The per-cell provenance a row carries on the wire, one key per enrichment column. +#: ⛔ A ROW KEY RATHER THAN A SIBLING MAP, and the choice is load-bearing rather than cosmetic. +#: A `{colId: {pid: state}}` map beside `rows` would need a new PROP on `RecordDetail` and a new +#: argument at `CustomerGrid`'s call site, both in another lane's fence, to reach the two surfaces +#: that must paint it. The row already carries `_created`, `lat` and `lon` for exactly this +#: reason, so every reader already tolerates keys that are not columns, and both surfaces hold the +#: row already. ⚠ COLLISION-PROOF BY CONSTRUCTION: `_clean_field` strips leading underscores off +#: every field key, so no column can ever be called `_ai_*`. +AI_STATE_PREFIX = "_ai_" + + +def _stamp_ai_states(table_key, fields, rows, st=None): + """Add `_ai_` to each row for every `ai_enrich` column. Mutates and returns `rows`. + + ⛔ A PROJECTION, NOT THE MARK SET. The stratum holds a hash, a model, a timestamp, a token + count and an error per cell; a browser needs ONE WORD to paint a state, and shipping the rest + would grow this payload by a dict per enriched cell for data no reader reads. The vocabulary + is `agent`/`human`/`stale`/`error` (`api/ai_enrich.py::cell_state`), which is also what the + RUNNER obeys, so the badge and the behaviour cannot disagree about whose cell it is. + + ⚠ AN ABSENT KEY MEANS `empty`, and only non-empty states are stamped: a table with no + enrichment column is untouched, and a freshly created column adds nothing until something + runs. ⛔ `stale` is DERIVED here rather than stored, so it is computed against TODAY'S row + instead of against whatever was true when the value was written. + """ + cols = [f for f in (fields or []) if str(f.get("type") or "") == "ai_enrich"] + if not cols: + return rows + import ai_enrich as _ae + for field in cols: + col = str(field.get("key") or "") + marks = _ut().ai_enrich_marks(table_key, col, st=st) + cfg = field.get("aiEnrich") if isinstance(field.get("aiEnrich"), dict) else {} + for row in (rows or []): + if not isinstance(row, dict): + continue + state = _ae.cell_state(marks.get(str(row.get("pid"))), cfg, row, col) + if state != "empty": + row[AI_STATE_PREFIX + col] = state + return rows + + +@router.get("/tables/{table_key}/rows/{pid}/fields/{fkey}") +def table_cell(table_key: str, pid: str, fkey: str, + session: Session = Depends(require_session)): + """ONE cell, whole — the other half of `_thin_json`. + + ⛔ WITHOUT THIS THE THINNING WOULD BE A CAP, and a cap on data somebody already paid a vendor + for is exactly what this product refuses everywhere else. The list ships a stand-in; the JSON + viewer opens this for the one row a person is actually reading. + + ⚠ SAME PERMISSION WALL AS THE LIST, reached the same way (`ut_assembly` resolves the session's + view of the table), so this cannot become a side door onto a table the caller may not open — + which is the failure a "just fetch the raw cell" helper invites. + ⚠ THE OVERLAY WINS, exactly as it does in `table_rows`: a user who has typed over a cell must + read back what they typed, not the definition value underneath it. + """ + g = ut_assembly(session, table_key) + field = next((f for f in (g.get("fields") or []) if str(f.get("key")) == str(fkey)), None) + if field is None: + raise err(404, "unknown field", f"{fkey!r} is not a column on this database") + row = next((r for r in g["rows_src"] if str(r.get("pid")) == str(pid)), None) + if row is None: + raise err(404, "unknown record", f"no record {pid!r} in this database") + overlay = ((g.get("ws") or {}).get("overlays") or {}).get(str(pid)) or {} + value = overlay.get(fkey, row.get(fkey)) + return {"table": table_key, "pid": str(pid), "field": str(fkey), + "value": "" if value is None else str(value)} + + +@router.post("/tables/{table_key}/rows", status_code=201) +def add_row(table_key: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Append a row — or RESTORE one under its old id (contract C-ADDROW / C-UNDO). + + ⚠ THE ANSWER IS ALWAYS THE ID THAT WAS STORED, never the one that was asked for. An undo + that requested `rid: 7` and got 12 because 7 had been re-used must find that out from the + response rather than assume; the client re-anchors on what came back. + """ + _records_or_refuse(session, table_key) + ut = _ut() + values = (body or {}).get("values") or {} + if not isinstance(values, dict): + raise err(400, "bad_values", "values must be an object of {fieldKey: value}") + try: + rid = ut.add_row(table_key, values, session.uname, st=session.runtime, + rid=(body or {}).get("rid")) + except Exception: + raise err(503, "store_unavailable", "the row was not saved. The store refused") + if rid is None: + # C3 (wave 25): `add_row` also refuses a profile cell that is not a handle, so the cap + # sentence alone would misdirect — the reader would go and count rows. Ask the same + # validator the law used rather than re-deciding here (one rule, two voices). + pf = ut.profile_field(table_key, st=session.runtime) + if pf and pf["key"] in values: + _h, ok = ut.normalize_profile(values[pf["key"]], pf["profile"].get("source")) + if not ok: + raise err(400, "refused", + f"{str(values[pf['key']])[:80]!r} is not an Instagram profile. " + f"{pf.get('label') or pf['key']!r} takes a handle (@name) or a " + f"profile link (instagram.com/name)") + raise err(400, "refused", + f"row refused. The table may be at its {ut.MAX_ROWS}-row cap") + _refresh_relations(session) + return {"rid": rid, "pid": int(rid)} + + +@router.post("/tables/{table_key}/rows/import", status_code=201) +def import_rows(table_key: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """⭐⭐ WAVE-29 T25 (owner item 6) — the IMPORT door: N mapped rows, ONE store write. + + Body: `{"rows": [{fieldKey: value, ...}, ...]}` — already MAPPED by the client's dialog, so a + spreadsheet column name never reaches the store. APPEND-ONLY in v1: every row is a new record, + nothing is matched or overwritten, and the dialog says so before the button is pressed. + + ⛔ COMPUTED COLUMNS ARE REFUSED HERE, NOT FILTERED. `is_computed_cell` is the same predicate + the cell wall uses (one evaluator for one question), and a rollup or formula key arriving in + an import is not a stray to be tidied away — it means the client offered a target it should + not have, and silently dropping it would leave the user looking for a column of values that + never arrived. The refusal names the column. + + ⛔ ATOMIC. `add_rows` writes nothing unless the whole batch fits under `MAX_ROWS` and every + profile cell validates, because a half-imported file is the worst outcome available: the user + cannot tell which rows landed without reconciling the spreadsheet by hand. + """ + _records_or_refuse(session, table_key) + ut = _ut() + rows_in = (body or {}).get("rows") + if not isinstance(rows_in, list) or not rows_in: + raise err(400, "bad_rows", "rows must be a non-empty array of {fieldKey: value} objects") + if any(not isinstance(r, dict) for r in rows_in): + raise err(400, "bad_rows", "every row must be an object of {fieldKey: value}") + defn = _defn_or_refuse(session, table_key) + by_key = {f["key"]: f for f in (defn.get("fields") or [])} + asked = {k for r in rows_in for k in r} + unknown = sorted(k for k in asked if k not in by_key) + if unknown: + raise err(400, "unknown_field", + f"this database has no column {unknown[0]!r}") + computed = sorted(k for k in asked if ut.is_computed_cell(by_key[k])) + if computed: + label = by_key[computed[0]].get("label") or computed[0] + raise err(400, "computed_field", + f"{label!r} is worked out from other columns, so it cannot be imported into") + # ⛔ W29-T81 — THE TYPE WALL, AND IT LIVES HERE BECAUSE THE ONLY OTHER ONE IS IN THE BROWSER. + # `coerceClipboardValue` refuses "seventeen-ish" at an `int` column in the dialog; curl, a + # second client, or a future importer met no wall at all and the string landed verbatim in a + # typed column, where the grid then painted it as a fabricated `0`. Refused whole and BEFORE + # `add_rows`, matching this door's own atomicity: a half-imported file is the outcome every + # rule on this route exists to prevent. The sentence names the row and the column, because + # "invalid value" sends somebody hunting through 2,000 lines of spreadsheet. + for index, row in enumerate(rows_in): + for key, value in row.items(): + why = ut.cell_type_refusal(by_key[key], value) + if why: + raise err(400, "bad_value", f"row {index + 1}: {why}. Nothing was imported") + try: + made = ut.add_rows(table_key, rows_in, session.uname, st=session.runtime) + except Exception: + raise err(503, "store_unavailable", "nothing was imported. The store refused") + if made is None: + raise err(400, "refused", + f"nothing was imported. {len(rows_in)} rows would take this database past " + f"its {ut.MAX_ROWS}-row cap, or a profile column rejected a value") + _refresh_relations(session) + return {"imported": len(made), "pids": [int(r) for r in made]} + + +# --------------------------------------------------------------------------------------------- +# THE SHARED FIELD SCHEMA (contract C-FIELD, owner ruling R2) +# --------------------------------------------------------------------------------------------- +# R2: a `ut_*` table's fields are the TABLE'S schema — everyone with access sees the same +# columns, the creator or an admin edits them, and a per-field `editRole` can open ONE column's +# definition to everyone without handing over the table. This supersedes wave 17's "fields are +# per-user" law for this path only; the connector scopes keep their own model. +# +# ⚠ WHY IT MATTERS BEYOND TIDINESS: a grid add-field on a `ut_` scope used to land in the +# per-user workspace overlay, which is why the automation editor's "Automation column" picker +# could not see a column the user had just created — it reads the DEFINITION. Same defect shape +# as the rename (item 6a): two places to look, and the surfaces disagreed silently. + +def _field_or_refuse(session, table_key, fkey=""): + """The schema wall. `_defn_or_refuse` first (404/403 on the table), then the per-field rule. + + ⭐ W33-T01 / D-213: definitions only. Everything read off `defn` here is `fields` and + `createdBy`; the returned value is DISCARDED by all three callers (they re-fetch what they + write through the `user_tables` write doors), so no projected snapshot survives into a write. + ⚠ It is a SCHEMA wall on a write route, not a row-write wall — the distinction contract C5 + draws is about the snapshot reaching a read-BACK, and this one does not escape the function.""" + defn = _defn_or_refuse(session, table_key, defs_only=True, scope_applied=True) + ut = _ut() + if not ut.is_user_table(table_key, st=session.runtime): + raise err(400, "not_a_user_table", + "only a user-created database has an editable schema. A connected source " + "owns its own columns") + # ⭐⭐ W36-T21 — A HIDDEN COLUMN IS NOT EDITABLE, AND THIS IS THE DOOR THAT HAD TO SAY SO. + # `EventCtx.hidden_keys` walls the events transport; the REST schema routes (rename, retype, + # delete a column) do not pass through it at all. Without this an account that could not SEE + # `unit_cost` could still DELETE it for the whole tenant — the loudest possible version of a + # wall that exists on one wire only. ⚠ Read the closure off the DECLARED fields, which is what + # `routes_admin` validates a stored `hiddenFields` list against. + if fkey and str(fkey) in _ut_hidden(session, table_key, defn.get("fields") or []): + raise err(403, "forbidden", + "this database has no column by that name that you may edit") + if fkey and not ut.may_edit_field(table_key, fkey, session.uname, session.admin, + st=session.runtime): + field = next((f for f in (defn.get("fields") or []) + if f.get("key") == str(fkey)), None) + if isinstance((field or {}).get("automation"), dict) \ + and field["automation"].get("preset") is True: + # ⚠ THIS BRANCH NARRATES `may_edit_field`, it does not re-decide (the `and not + # ut.may_edit_field` above is the wall). Said out loud because the sentence itself + # went stale on 2026-08-09: preset ROLLUPS became editable by owner ruling, so a + # blanket "pre-set fields are locked" would now be the server explaining a refusal + # it did not make — `preset_editable` is the one predicate that answers this. + raise err(403, "preset_field_locked", + "this is a pre-set column, so its name and type are fixed; you may sort, " + "filter or hide it, edit any Rollup column, and add your own columns") + raise err(403, "forbidden", "that column can only be changed by the database's creator " + "or an admin") + if not fkey and not (session.admin or defn.get("createdBy") == session.uname): + raise err(403, "forbidden", "only the database's creator or an admin can add a column") + return defn + + +@router.post("/tables/{table_key}/fields", status_code=201) +def add_field(table_key: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + _field_or_refuse(session, table_key) + ut = _ut() + field = ut.add_field(table_key, body or {}, st=session.runtime) + if not field: + # ⭐ D-46 CLOSED (wave 23) — the C8 flow law gets its OWN sentence. `add_field` answers + # None for every refusal, so this route said "check the name and type" to somebody whose + # name and type were fine and whose automation column named a flow that does not exist. + # A refusal that misdirects is worse than a bare 400: it sends the reader to look at the + # one thing that was never wrong. Checked HERE, in the route's own words, because the + # law itself stays enforced in `user_tables.flow_bound` — this narrates it, never + # re-implements it (a second copy of the rule is how two doors start disagreeing). + raise err(400, "refused", + _refusal_sentence(ut, session, body or {}, table_key=table_key)) + if field.get("type") == "link": + synced = ut.sync_reciprocal_link(table_key, field["key"], st=session.runtime) + field = synced.get("field") or field + # ⭐⭐ 2026-08-09 — `rollup` REFRESHES TOO, and the omission was invisible until this route + # became reachable for one. It was gated on `link` alone, while `patch_field` and + # `delete_field` next door refresh unconditionally — so a newly created Rollup got its first + # fold from `_store_resync_loop`, which sleeps 1800 s BEFORE its first pass (D-107's shape). + # The user would have created the column, watched a 201 come back, and read a blank cell for + # half an hour: *"the Rollup doesn't work"*, arriving through the door opened to fix it. + # ⚠ Still conditional rather than unconditional: a pass deep-copies every table and row in + # the tenant, and adding a text column has nothing to fold. The condition is now "is this + # field relational", which is the question that was always meant. + if field.get("type") in ("link", "rollup"): + _refresh_relations(session) + return _with_dropped(ut, {"field": field}, body) + + +def _with_dropped(ut, out, body): + """Attach the NAMED list of config keys the validator did not keep (wave 34, R13 / W34-T51). + + ⛔⛔ THIS EXISTS BECAUSE THE VALIDATOR HAS NO ERROR CHANNEL AND CANNOT GROW ONE. Every bag + cleaner in `core/user_tables.py` returns `dict | None` and drops unknown keys in silence, and + `verify_fields_contract` asserts that they do -- so the drop is correct and the SILENCE is the + defect. T51's contract is that an unknown config key is dropped **and named**, so the naming + rides the response beside the accepted field rather than inside the validator. + + ⚠ OMITTED WHEN EMPTY, deliberately: an always-present `dropped: []` teaches every reader to + ignore the key, which is how a report stops being read before it stops being true. + """ + dropped = ut.ai_enrich_dropped_keys((body or {}).get("aiEnrich")) + if dropped: + out = dict(out) + out["dropped"] = dropped + return out + + +def _refusal_sentence(ut, session, body, table_key="", fkey=""): + """Why was this column refused? The specific reason when we can name one, the general list + otherwise — never a specific-sounding guess.""" + # ⭐ WAVE-34 (R13): the enrichment column's own sentence, named BEFORE the automation bag + # below. `_clean_field` DERIVES `field.automation` for this kind, so a refused enrichment + # column would otherwise be explained by the flow law -- "pick a flow, or make this an + # ordinary column" -- which is the D-46 misdirection exactly, pointing at a control the user + # never touched. + if str((body or {}).get("type") or "").strip().lower() == "ai_enrich": + bag = (body or {}).get("aiEnrich") + if not isinstance(bag, dict) or not str(bag.get("prompt") or "").strip(): + return ("an AI enrichment column needs a prompt. It is the only thing that can " + "produce a value here, so a column without one would stay empty forever") + # ⭐ C3 (wave 25, R7): the one-profile-per-table refusal NAMES THE EXISTING COLUMN, which is + # what the contract asks for and what makes it actionable — "at most one" sends the reader + # hunting through a 40-column schema for a flag they cannot see from the header. + if isinstance((body or {}).get("profile"), dict): + # ⚠ THE TYPE IS NAMED FIRST, and the order is the point. Both refusals can be true at + # once (an `int` profile column on a table that already has a profile column), and the + # TYPE is the one that is wrong about what the caller just sent — unconditionally, no + # matter what else is on the table. Answering "you already have one" to somebody whose + # real mistake was the column type sends them to fix the wrong thing, which is the + # misdirection D-46 closed one door over. + if str((body or {}).get("type") or "text").strip().lower() != "text": + return ("a profile column is a flag on an ordinary TEXT column. It validates what " + "is typed into it, which it can only do for text") + existing = ut.profile_field(table_key, st=session.runtime) if table_key else None + if existing and existing.get("key") != str(fkey): + return (f"this database already has a profile column: " + f"{existing.get('label') or existing.get('key')!r}. A database has at most " + f"one, so the automation knows which handle to enrich; edit that column, or " + f"take the flag off it first") + bag = (body or {}).get("automation") + if isinstance(bag, dict): + flow = str(bag.get("flowId") or "").strip() + if not flow: + return ("an automation column has to name the automation that fills it. Pick a " + "flow, or make this an ordinary column") + if not ut.flow_bound(bag, st=session.runtime): + return (f"this column names automation {flow!r}, which does not exist in this " + f"workspace. It may have been deleted; pick a flow that is still there") + kind = str((body or {}).get("type") or "").strip() + if kind and kind not in ut.UT_FIELD_TYPES: + return (f"{kind!r} is not a column type here (types: " + f"{', '.join(sorted(ut.UT_FIELD_TYPES))})") + return (f"the column was refused. Check the name and type, or the table may be at its " + f"{ut.MAX_FIELDS}-column cap (types: {', '.join(sorted(ut.UT_FIELD_TYPES))})") + + +@router.patch("/tables/{table_key}/fields/{fkey}") +def patch_field(table_key: str, fkey: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Edit one column's definition, and MIGRATE its values when options are renamed. + + ⛔ A CHOICE RENAME IS AN EXPLICIT MAPPING, NEVER A DIFF (contract C-RENAME). `{renames: + [{from, to}]}` arrives alongside the new options list, because a diff cannot tell "renamed + Blue to Navy" from "deleted Blue, added Navy" — and guessing wrong empties the column and + every saved view that filtered on it. + """ + _field_or_refuse(session, table_key, fkey) + ut = _ut() + body = body or {} + migrated = None + renames = body.get("renames") + if renames: + try: + migrated = ut.rename_choice_values(table_key, fkey, renames, st=session.runtime) + except Exception: + raise err(503, "store_unavailable", "the rename did not land. Try again") + # The per-user workspace strata and any view filter naming the old value are the OTHER + # half of C-RENAME and belong to `core.table_store`. Called only if it is there: an + # enumerator's mirror waits for its counterpart rather than guessing at its shape, and a + # missing counterpart must not lose the half that DID land. + try: + import core.table_store as table_store + fn = getattr(table_store, "rename_choice_values", None) + if callable(fn): + fn(table_key, fkey, renames, st=session.runtime) + migrated = dict(migrated or {}, workspace=True) + except Exception: # noqa: BLE001 + migrated = dict(migrated or {}, workspace=False) + field = ut.patch_field(table_key, fkey, body, st=session.runtime) + if not field: + # C3: the same narrator as `add_field`. Flagging a SECOND column is the same act as + # adding one, so it must get the same sentence naming the column that already holds the + # flag — a patch that answered "check the name and type" would send the reader to the + # one thing that was never wrong (the D-46 lesson, one door over). + raise err(400, "refused", + _refusal_sentence(ut, session, body, table_key=table_key, fkey=fkey)) + synced = ut.sync_reciprocal_link(table_key, fkey, st=session.runtime) + field = synced.get("field") or field + _refresh_relations(session) + out = {"field": field} + if migrated is not None: + out["migrated"] = migrated + return _with_dropped(ut, out, body) + + +def _fire_on_change(table_key, pid, changed, session): + """Run any `on_change` enrichment column whose prompt names a cell that just moved. + + ⛔ ONE DEFINITION READ FOR THE WHOLE WRITE, and that is the point rather than an optimisation. + `automation_engine.grid_hook` calls `all_definitions(st)` once PER EVENT, which turns a + 20,000-row import into 20,000 whole-document reads on the single process this product runs + (`D-134`, and `W34-T54`'s own `how:` says not to rebuild it). `on_change_fields` is pure and + takes the definition, so this reads once and asks about every column. + + ⛔ AND IT NEVER FAILS THE WRITE. The cell edit has already succeeded and been acknowledged; + an enrichment that could not run is a missing value, not a lost edit, and the run's own report + carries the reason. ⚠ It is also deliberately SYNCHRONOUS and bounded to this one row: a + fan-out here would put a vendor call on the critical path of every keystroke-commit. + """ + import ai_enrich as _ae + + try: + defn = _ut().get(table_key, st=session.runtime) or {} + wanted = _ae.on_change_fields(defn, changed.keys()) + for field in wanted: + # ⭐ W35-T41 / C7 — `user` is the usage ledger's attribution. An on-change run is still + # somebody's edit spending somebody's tokens, so it is booked against the person who + # typed rather than left unattributed. + _ae.run_field(table_key, field["key"], st=session.runtime, rows=[str(pid)], + user=session.uname) + except Exception: # noqa: BLE001 + pass + + +@router.post("/tables/{table_key}/fields/{fkey}/enrich") +def enrich_field(table_key: str, fkey: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Run an AI enrichment column (wave 34, owner ruling R13). Returns the run's own REPORT. + + ⛔ THIS DOOR SPENDS MONEY, so it rides the same wall every other schema write rides + (`_field_or_refuse`) rather than a looser one of its own. A read-only viewer cannot bill the + tenant by opening a grid. + + `{"rows": ["3"]}` is a MANUAL run of exactly those records: a person asked, in front of the + value being replaced, so it skips the `overwrite` policy. An ABSENT `rows` is the automatic + plan, where the policy and the never-overwrite-a-human law both apply. The two are one + function with one flag, not two runners. + + ⚠ THE REPORT IS THE PRODUCT, not a status code. It carries `filled`, `failed`, `skipped` by + reason, `tokens` spent, the provider, per-row errors, and `limit` (R6's second sentence: a + ceiling that stopped the run names its cause and a remedy). A 200 with `filled: 0` and a + populated `skipped` is a correct, informative answer, and the client must render it rather + than treat it as success. + """ + _field_or_refuse(session, table_key, fkey) + import ai_enrich as _ae + + rows = (body or {}).get("rows") + if rows is not None and not isinstance(rows, list): + raise err(400, "bad_rows", "`rows` must be a list of record ids, or absent to run the " + "rows this column's own settings choose") + # The caller's permitted pool, the same one the row doors use. A named row outside it is + # dropped rather than refused: a stale client naming a record that has been deleted or moved + # out of scope should not fail a run over the rows it can legitimately fill. + if rows is not None: + allowed = {str(p) for p in scoped_pids(session, table_key)[0]} + rows = [str(r) for r in rows if str(r) in allowed] + # ⭐ `W34-T54`'s bulk menu is this one field: "Rows never filled" sends `blank`, "All rows" + # sends `always`. Anything else falls back to the column's own saved policy rather than to a + # default, so a typo cannot quietly widen what a run touches. + report = _ae.run_field(table_key, fkey, st=session.runtime, rows=rows, + policy=str((body or {}).get("scope") or "") or None, + # A named row set through THIS door is a person asking. + manual=rows is not None, + # ⭐ W35-T41 / C7 — the usage ledger's attribution. + user=session.uname) + if report.get("problem"): + # A run that could not start at all is not a 200: nothing was attempted, nothing was + # spent, and the reason is actionable (no provider configured, or the wrong column). + raise err(400, "enrich_refused", str(report["problem"])) + return report + + +@router.delete("/tables/{table_key}/fields/{fkey}") +def delete_field(table_key: str, fkey: str, session: Session = Depends(require_session)): + _field_or_refuse(session, table_key, fkey) + if not _ut().delete_field(table_key, fkey, st=session.runtime): + raise err(400, "refused", + "that column could not be removed. A database must keep at least one") + _refresh_relations(session) + return {"deleted": fkey} + + +@router.delete("/tables/{table_key}/rows/{rid}") +def delete_row(table_key: str, rid: str, session: Session = Depends(require_session)): + # ⭐⭐ W31 QA — ONE SNAPSHOT FOR THE WHOLE DELETE, and the owner reported what it cost. + # Owner, verbatim (2026-08-13): *"it still takes forever to delete a record from TT Profile."* + # A single DELETE was FIVE whole-document deep copies before the commit even began — three in + # the guard (`get` + `may_open` + `records_mutable`) and two more inside + # `core.user_tables.delete_row` (`is_user_table` + `records_mutable` again). MEASURED: 2 reads + # cost 45 ms on an 0.8 MB fixture, and tenant #0's `user_tables` document is **28.5 MB** + # (D-185), so the guard alone was seconds of copying to answer questions about one row. + # ⚠ THE LEND IS READ-ONLY AND THE WRITE STILL GOES THROUGH THE REAL RUNTIME — `_Lent` + # `__getattr__`-passes `update` straight to it, and `_drop` runs against the LIVE document + # under the store lock, so a lent snapshot can never be the thing written back. + # ⚠ `flush='sync'` IS DELIBERATELY UNTOUCHED (D-118): nobody spams a delete, and an eventually + # consistent delete is indistinguishable from one that did not work. This makes the guard + # cheap; it does not make the commit optimistic. + lent = _ut().lend(session.runtime) + _records_or_refuse(session, table_key, st=lent) + # ⭐⭐ W36-T21 — THE ROW SCOPE, ON THE DELETE DOOR. `patch_row` has asked this since it was + # written (`pid not in g["pids"]` -> 403) and this door never did, because until now every + # account that could open a `ut_*` database could see every row of it. The moment an + # administrator can row-scope one, "may not SEE row 5" and "may DELETE row 5" become two + # different answers unless this is here — and delete is the one that cannot be undone. + # ⚠ Guarded on `row_scope_applies` so an unscoped account pays nothing: for them the pid set + # is every row and the question has one answer. + import core.perm_scope as _ps + if _ps.row_scope_applies(session.user, table_key): + _pids, _f, _d = scoped_pids(session, table_key) + if not str(rid).isdigit() or int(rid) not in _pids: + raise err(403, "out_of_scope", "that row is not in this database") + try: + ok = _ut().delete_row(table_key, rid, st=lent) + except Exception: + raise err(503, "store_unavailable", "the delete did not land. Try again") + if not ok: + raise err(400, "refused", "rows can only be deleted from user-created databases") + _refresh_relations(session) + return {"ok": True} + + +@router.patch("/tables/{table_key}/rows/{pid}") +def patch_row(table_key: str, pid: int, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Cell edits — the products PATCH on the user-table ctx. Routed through + `core.grid_events.handle_one` so truncation and permission rules stay ONE implementation; + the accepted values are read BACK from the bucket, never echoed from the request.""" + from core import grid_events + + updates = dict(body or {}) + if not updates: + raise err(400, "empty_patch", "no fields to update") + _records_or_refuse(session, table_key) + g = ut_assembly(session, table_key, consume_corrections=False) + if pid not in g["pids"]: + raise err(403, "out_of_scope", "that row is not in this database") + ctx = grid_events.EventCtx( + uname=session.uname, allowed_pids=g["pids"], fields=g["fields"], + admin=session.admin, fallback_ws=None, seen_ids={}, + # ⭐⭐ W36-T21 — the assembly's OWN closure, not an empty set. `g["fields"]` is already + # stripped, but `grid_events` asks `ctx.hidden_keys` by name: a PATCH naming a hidden + # column would otherwise be accepted on a payload that never showed it. + hidden_keys=g.get("hidden") or frozenset(), + st=session.runtime, # R6b (D-16) + scope_key=table_key, table=_ops(session, table_key)) + try: + grid_events.handle_one( + {"id": f"patch:{table_key}:{pid}:{time.time_ns()}", "type": "overlay_patch", + "pid": pid, "updates": updates}, ctx) + except grid_events.StoreUnavailable: + raise err(503, "store_unavailable", + "the tenant store is unavailable. Your change was not saved") + # ⚠ THE READ-BACK IS THE DEFINITION ROW, AND ON A `ut_` SCOPE THAT IS THE WHOLE OF IT. + # + # ⛔ CORRECTED, wave-29 T22 (owner item 2a): this note used to say "THE READ-BACK SPANS BOTH + # STRATA, and it has to (wave 25, C3-A1)" while the two lines under it read exactly one bucket. + # It was true of the wave-25 world it was written in — an ordinary cell landed in the caller's + # OVERLAY and only a PROFILE cell wrote through — and it stayed after `grid_events` began + # routing EVERY accepted cell on a `ut_` scope to `user_tables.patch_cells` + # (`grid_events.py:1979-1984`). There is no second stratum this read is missing; a docstring + # claiming otherwise is what makes the next reader look for a merge bug that is not here. + # ⚠ Legacy overlay values, written before that routing existed, are still merged for DISPLAY + # by `table_rows` (:587-595) — display only, and deliberately not re-asserted here: `_took` + # asks whether THIS write landed, and this write goes to the definition. + stored = dict(((_ut().get(table_key, st=session.runtime) or {}).get("rows") or {}) + .get(str(pid)) or {}) + accepted = {k: stored.get(k) for k in updates if k in stored} + + def _took(k): + """Did the cell TAKE this write? Normally that is "stored == asked". + + ⚠ A PROFILE COLUMN CANONICALISES, so "stored != asked" is its NORMAL success: `@Nurilab` + and `instagram.com/nurilab` both store `nurilab`. Reporting those as refused would tell + the client to roll back a write that landed. But it cannot simply be exempted either — + a junk handle leaves the OLD value sitting in `stored`, which would then read as + accepted. So the question asked is the exact one: **is what is stored the canonical form + of what was asked?** Anything else is a genuine refusal. + """ + if k not in accepted: + return False + want = str(updates[k]) + if stored.get(k) == want: + return True + pf = _ut().profile_field(table_key, st=session.runtime) + if pf and pf["key"] == k: + handle, ok = _ut().normalize_profile(want, pf["profile"].get("source")) + return bool(ok) and stored.get(k) == handle + return False + + refused = sorted(k for k in updates if not _took(k)) + # ⭐⭐ WAVE-34 (R13) — THE HUMAN-EDIT STAMP, AND IT HAS TO HAPPEN HERE. "Did a person write + # this cell?" is not recoverable from the value afterwards, so the only place to record it is + # the door a person writes through. `ai_enrich_may_write` then refuses to let any automatic + # run overwrite it, whatever the column's `overwrite` policy says. + # ⚠ STAMPED FROM THE CELLS THAT ACTUALLY TOOK, never from what was asked: marking a refused + # write `human` would freeze a cell against the agent on the strength of an edit that never + # landed. `note_human_edit` filters to the enrichment columns itself and is a no-op otherwise. + took = {k: v for k, v in accepted.items() if k not in refused} + if took: + try: + _ut().note_human_edit(table_key, took, pid, st=session.runtime) + except Exception: # noqa: BLE001 + # Provenance is metadata about a write that has already succeeded. Failing the + # request here would tell the user their edit was lost when it was not. + pass + _fire_on_change(table_key, pid, took, session) + out = {"pid": pid, "updates": accepted} + if refused: + out["refused"] = refused + # ⭐ R6: the cells the SERVER changed that the client never typed — the preset cells a + # profile blank cleared. Without this the grid keeps painting a stale follower count under + # an empty handle until something else forces a refetch, which is the NO-BLIP LAW's other + # half: the client may keep only what the server actually took, and must be TOLD what else + # moved. Derived by diffing this row against what was asked for, so it cannot drift from + # whatever the clear rule decides to touch. + also = {k: v for k, v in stored.items() if k not in updates and str(v or "") == ""} + cleared = sorted(k for k in also if k in _ut().PROFILE_PRESET_KEYS) + if cleared: + out["cleared"] = cleared + # ⭐⭐ R9's SECOND RE-ARM DOOR — the one call that makes `engine.clear_gone` live (wave 28, + # amendment A5; SESSION B built and gated the function and correctly declared it INERT until + # this line existed, citing [[flag-shipped-without-its-writer]]). + # + # ⛔ R9 makes a `not_found` handle a TOMBSTONE, not a 30-day backoff: nothing re-buys a dead + # account on a timer any more. Door 1 — correcting the handle — needs no wiring, because the + # verdict is keyed on `(platform, handle)` and a corrected handle simply is not the verdict we + # recorded. THIS is door 2: a human re-typing the SAME handle, which is how somebody says "try + # it again, the account is back". Without this call that person has no way back at all, and + # the failure costs nothing and raises nothing — so no spend-shaped test would ever find it. + # + # ⚠ GATED ON `updates`, NOT ON `accepted`: re-typing the identical value is the whole case this + # door exists for, and a no-op write can be filtered out of `accepted`. What matters is that a + # human touched the handle cell. + # ⚠ NOT a bare `except: pass`. A swallowed AttributeError here would be exactly the optional- + # prop silence this wiring exists to prevent — if the engine ever loses `clear_gone`, that must + # be readable in the log rather than degrade into "the re-arm quietly stopped working". + _pf = _ut().profile_field(table_key, st=session.runtime) + if _pf and _pf["key"] in updates: + try: + import automation_engine as _engine + _engine.clear_gone(session.runtime, table_key, stored.get(_pf["key"])) + except Exception as e: # noqa: BLE001 + print(f"[aios-api] clear_gone failed: {type(e).__name__}: {e}") + _refresh_relations(session) + return out + + +# ── CONTRACT C1 (W36-T20): THE MIRROR READER ────────────────────────────────────────────────── +# ⭐⭐ R6. `core.perm_scope.scoped_table` is the ONE door to any database's rows and it answers for +# a MATERIALISED `ut_*` table entirely on its own — deliberately, so a cold process (E's sandbox +# subprocess, a worker, a gate) that never imported a route still gets the right answer. A +# READ-THROUGH grid is the one arm it cannot serve alone: those ten databases store no rows in the +# tenant document at all, and their rows live in the DuckDB mirror behind `routes_odoo_tables`, +# two layers above `core`. +# +# ⛔ ONE FETCH, NOT A SECOND ONE. This hands C1 the SAME `_read_through_rows` that `scoped_pool` +# and `scoped_pids` already share, so the rows a script sees through C1 are byte-for-byte the rows +# the grid sees — by construction, not by a second query that agrees today (W31-T20's argument, one +# caller further out). +# +# ⚠ AND `TooBigToMaterialise` BECOMES A REPORTED REFUSAL, NEVER A SHORT ANSWER. Standing rule 1's +# second sentence, and the shape is `_PID_SCOPE_LIMIT`'s so nothing downstream needs a second +# vocabulary for it. +def _c1_mirror_rows(table_key, field_keys, st): + """C1's mirror arm: `perm_scope.register_mirror`'s reader over `_read_through_rows`.""" + import core.perm_scope as perm_scope + + try: + return _read_through_rows(table_key, field_keys, rt=st) + except _too_big() as e: + raise perm_scope.Unresolvable( + cause=str(e), + recommendation=_PID_SCOPE_LIMIT["recommendation"], + subject="rows", effect="unresolved") from e + except RuntimeError as e: + raise perm_scope.Unresolvable( + subject="rows", effect="unreadable", cause=str(e), + recommendation="the connector mirror is not ready on this process; retry once the " + "store has finished opening") from e + + +def _register_mirror(): + """Declare the mirror reader to C1. Called at import; returns True once it is registered.""" + import core.perm_scope as perm_scope + return perm_scope.register_mirror(_c1_mirror_rows) + + +_C1_MIRROR = _register_mirror() diff --git a/api/routes_web_agent.py b/api/routes_web_agent.py index 12364aa7911fe7e0042054e3a7f07e050fae7700..57453ded998810e7046aa5d14acae4d739fd235b 100644 --- a/api/routes_web_agent.py +++ b/api/routes_web_agent.py @@ -1,84 +1,84 @@ -"""routes_web_agent.py — the door that lets a person TEST the web agent (wave 31, R10 / D-51). - -The owner's words are the whole reason this file exists: *"we already laid the foundation of this -but never test anything."* The capability is otherwise reachable only from inside an automation -run, which means the first person to discover it is broken is a customer at 3am. - - GET /api/v1/web-agent/capability any session — can this deployment run a web step, and - if not, the SENTENCE saying why - POST /api/v1/web-agent/test ADMIN — run one real `web_read` and show what - came back, or the sentence - -⛔ NEITHER ROUTE IS THE SEAM. `automation_engine` calls `web_agent.run_step` directly (contract -C5); these are an operator surface over the same function, so a green test here and a red run -there cannot disagree about anything except the input. - -⚠ `POST /test` BLOCKS FOR ~10-30 s and COSTS A FRACTION OF A CENT. It is `def`, not `async def`, -so FastAPI runs it in the threadpool and one test cannot stall the event loop for everybody. - -⚠ ON THE URL IT WILL FETCH: the fetch happens inside an ephemeral HF Job on Hugging Face's -network, never from this server, so this is not a door into our own infrastructure. It is still -admin-gated, because it spends money and because D-51 §5's authorisation posture ("only systems -the tenant is authorised to use, at their instruction") is not something an ordinary member -should be able to commit the tenant to. - -⭐ MOUNTED, and the gate is what says so rather than this sentence: `main.py:80` imports it and -`main.py:316` includes it, and `verify_web_agent.py` asserts the route answers rather than trusting -either line. This paragraph read "NOT MOUNTED YET" for a whole wave after the mount landed, which -is the same class of stale claim as a green gate on a router nobody wired: three finished routers -once shipped 404-dead behind entirely green gates, and prose is not the control that stops it. -""" -from fastapi import APIRouter, Body, Depends - -import web_agent -from deps import Session, require_session, err -from routes_admin import admin_gate - -router = APIRouter(prefix="/api/v1") - -MAX_URL = 2000 -MAX_SELECTOR = 400 - - -@router.get("/web-agent/capability") -def web_agent_capability(session: Session = Depends(require_session)): - """Can a web step run here at all? Configuration, not liveness — see `web_agent.capability`. - - Deliberately NARROW: it answers the question a UI needs ("may I offer this, and what do I say - if not") and withholds the deployment detail (namespace, which token key, the image) that - only an operator has any use for. Nothing here is ever a credential. - """ - cap = web_agent.capability() - return {"ready": bool(cap["ready"]), "reason": cap["reason"], - "runnableKinds": cap["runnableKinds"], "profile": cap["profile"]} - - -@router.post("/web-agent/test") -def web_agent_test(session: Session = Depends(admin_gate), body: dict = Body(...)): - """Run ONE real `web_read` and report exactly what the seam returned. - - The response mirrors the seam's own contract rather than flattening it: `ok` plus a `value`, - or `ok:false` plus the SENTENCE. A test surface that turns a named failure into "something - went wrong" would hide the one thing it exists to show. - """ - url = str(body.get("url") or "").strip() - selector = str(body.get("selector") or "").strip() - if len(url) > MAX_URL or len(selector) > MAX_SELECTOR: - raise err(400, "too_long", "the URL or selector is longer than this door accepts") - - step = {"kind": "web_read", "id": "test", "url": url, "selector": selector, - "attr": (body.get("attr") or "text"), "all": bool(body.get("all")), - "timeoutMs": int(body.get("timeoutMs") or 20000)} - if body.get("waitFor"): - step["waitFor"] = str(body["waitFor"]) - - notes = [] - result, error = web_agent.run_step( - step, {"tenant": session.tenant, "runId": f"test-{session.tenant}-{session.uname}", - "log": notes.append}) - if error: - # 200 with `ok:false`, not a 4xx: the request was well-formed and the ANSWER is that the - # web step did not succeed. A 500 here would make an ordinary "the selector matched - # nothing" look like a server fault. - return {"ok": False, "error": error, "log": notes[-6:]} - return {"ok": True, "result": result, "log": notes[-6:]} +"""routes_web_agent.py — the door that lets a person TEST the web agent (wave 31, R10 / D-51). + +The owner's words are the whole reason this file exists: *"we already laid the foundation of this +but never test anything."* The capability is otherwise reachable only from inside an automation +run, which means the first person to discover it is broken is a customer at 3am. + + GET /api/v1/web-agent/capability any session — can this deployment run a web step, and + if not, the SENTENCE saying why + POST /api/v1/web-agent/test ADMIN — run one real `web_read` and show what + came back, or the sentence + +⛔ NEITHER ROUTE IS THE SEAM. `automation_engine` calls `web_agent.run_step` directly (contract +C5); these are an operator surface over the same function, so a green test here and a red run +there cannot disagree about anything except the input. + +⚠ `POST /test` BLOCKS FOR ~10-30 s and COSTS A FRACTION OF A CENT. It is `def`, not `async def`, +so FastAPI runs it in the threadpool and one test cannot stall the event loop for everybody. + +⚠ ON THE URL IT WILL FETCH: the fetch happens inside an ephemeral HF Job on Hugging Face's +network, never from this server, so this is not a door into our own infrastructure. It is still +admin-gated, because it spends money and because D-51 §5's authorisation posture ("only systems +the tenant is authorised to use, at their instruction") is not something an ordinary member +should be able to commit the tenant to. + +⭐ MOUNTED, and the gate is what says so rather than this sentence: `main.py:80` imports it and +`main.py:316` includes it, and `verify_web_agent.py` asserts the route answers rather than trusting +either line. This paragraph read "NOT MOUNTED YET" for a whole wave after the mount landed, which +is the same class of stale claim as a green gate on a router nobody wired: three finished routers +once shipped 404-dead behind entirely green gates, and prose is not the control that stops it. +""" +from fastapi import APIRouter, Body, Depends + +import web_agent +from deps import Session, require_session, err +from routes_admin import admin_gate + +router = APIRouter(prefix="/api/v1") + +MAX_URL = 2000 +MAX_SELECTOR = 400 + + +@router.get("/web-agent/capability") +def web_agent_capability(session: Session = Depends(require_session)): + """Can a web step run here at all? Configuration, not liveness — see `web_agent.capability`. + + Deliberately NARROW: it answers the question a UI needs ("may I offer this, and what do I say + if not") and withholds the deployment detail (namespace, which token key, the image) that + only an operator has any use for. Nothing here is ever a credential. + """ + cap = web_agent.capability() + return {"ready": bool(cap["ready"]), "reason": cap["reason"], + "runnableKinds": cap["runnableKinds"], "profile": cap["profile"]} + + +@router.post("/web-agent/test") +def web_agent_test(session: Session = Depends(admin_gate), body: dict = Body(...)): + """Run ONE real `web_read` and report exactly what the seam returned. + + The response mirrors the seam's own contract rather than flattening it: `ok` plus a `value`, + or `ok:false` plus the SENTENCE. A test surface that turns a named failure into "something + went wrong" would hide the one thing it exists to show. + """ + url = str(body.get("url") or "").strip() + selector = str(body.get("selector") or "").strip() + if len(url) > MAX_URL or len(selector) > MAX_SELECTOR: + raise err(400, "too_long", "the URL or selector is longer than this door accepts") + + step = {"kind": "web_read", "id": "test", "url": url, "selector": selector, + "attr": (body.get("attr") or "text"), "all": bool(body.get("all")), + "timeoutMs": int(body.get("timeoutMs") or 20000)} + if body.get("waitFor"): + step["waitFor"] = str(body["waitFor"]) + + notes = [] + result, error = web_agent.run_step( + step, {"tenant": session.tenant, "runId": f"test-{session.tenant}-{session.uname}", + "log": notes.append}) + if error: + # 200 with `ok:false`, not a 4xx: the request was well-formed and the ANSWER is that the + # web step did not succeed. A 500 here would make an ordinary "the selector matched + # nothing" look like a server fault. + return {"ok": False, "error": error, "log": notes[-6:]} + return {"ok": True, "result": result, "log": notes[-6:]} diff --git a/platform/aios_grid.py b/platform/aios_grid.py index 88cb8a8a615a0632d3002e8cdd709a7018b00539..a512ac930242fe018480ae678aec50847a39c263 100644 --- a/platform/aios_grid.py +++ b/platform/aios_grid.py @@ -1,2645 +1,2645 @@ -"""aios_grid — embed the AIOS React/glide Airtable-style grid inside Streamlit. - -This is REUSABLE MODULE INFRASTRUCTURE: the same self-contained grid that runs as the -standalone aios-web app is inlined into a single HTML file and hosted inside a Streamlit -component. The React bundle reads its data from `window.__AIOS_GRID__` (an object -`{fields, rows}`) that we inject into the page before the app's module script runs -— so there is NO backend and NO /api call in the Streamlit container; the browser only ever -sees the derived JSON we hand it. - -Usage (from any page): - import aios_grid - aios_grid.render(aios_grid.rows_from_pool(pool_rows), aios_grid.FIELDS) - -Design notes: - * ZERO app-internal imports (no `modules.*` / `core.*`). FIELDS is a literal and - `rows_from_pool` takes already-built rows — so this helper is tenant/module-agnostic and - sidesteps deploy_hf.py's import guard entirely. - * The built HTML ships as `aios_grid_embed.html` in THIS directory (added to - deploy_hf.py INCLUDE). Build it with `npm run build:embed` in aios-web/web, then copy - dist-embed/index.html -> platform/aios_grid_embed.html (see build_embed.py). - For LOCAL dev before that copy, we fall back to reading dist-embed/index.html directly. - * The preferred host is a Streamlit Components v1 bridge. It sends data/view/schema args - into the React app and returns guarded events for persistence in the tenant store. The old - injected-HTML path remains a read/local-write fallback when component assets are absent. -""" -import json -import math -import re -from pathlib import Path - -_HERE = Path(__file__).resolve().parent - -# Where the inlined single-file build lives. FIRST match wins: -# 1. the shipped copy in this dir (what deploy_hf.py uploads to the Space) -# 2. the raw build output in the sibling aios-web tree (local dev, pre-copy) -_EMBED_CANDIDATES = [ - _HERE / "aios_grid_embed.html", - _HERE.parent / "aios-web" / "web" / "dist-embed" / "index.html", -] -_COMPONENT_CANDIDATES = [ - _HERE / "aios_grid_component", - _HERE.parent / "aios-web" / "web" / "dist-embed", -] -_DECLARED_COMPONENTS = {} - -# --- the FIELD CONTRACT — loaded from the CANONICAL source `aios_grid_fields.json` in THIS -# directory (the SINGLE source of truth, shared with aios-web/api/main.py). source='odoo' is -# READ-ONLY; source='overlay' is the editable stratum (notes/tags) that lives OUTSIDE Odoo. -# -# Why a sibling JSON and NOT an import: aios_grid.py's contract is ZERO app-internal imports so -# it stays tenant/module-agnostic and sidesteps deploy_hf.py's import guard. A JSON next to the -# module preserves that exactly — no import, no guard interaction — while still single-sourcing -# the values (a build-time copy from another tree would reintroduce the drift we're removing). -# The file ships to the Space via deploy_hf.py INCLUDE. Edit the JSON, then run -# aios-web/verify_fields_contract.py. --- -_FIELDS_PATH = _HERE / "aios_grid_fields.json" - - -def _load_fields(): - if not _FIELDS_PATH.is_file(): - raise FileNotFoundError( - f"aios_grid: canonical field contract missing at {_FIELDS_PATH}. It is the single " - "source of truth for the grid schema and MUST ship (deploy_hf.py INCLUDE lists it)." - ) - doc = json.loads(_FIELDS_PATH.read_text(encoding="utf-8")) - fields = doc.get("fields") if isinstance(doc, dict) else doc - if not isinstance(fields, list) or not fields: - raise ValueError(f"aios_grid: {_FIELDS_PATH} has no 'fields' list.") - return fields - - -FIELDS = _load_fields() - - -def product_fields(): - """The PRODUCT table's canonical contract (wave 15 C-TOPIC, `product_data` key in the same - JSON). A separate accessor rather than a second module constant so the one file-read and the - one failure mode stay shared with `FIELDS`.""" - doc = json.loads(_FIELDS_PATH.read_text(encoding="utf-8")) - fields = (doc.get("product_data") or {}).get("fields") - if not isinstance(fields, list) or not fields: - raise ValueError(f"aios_grid: {_FIELDS_PATH} has no product_data.fields list.") - return fields - -# text/date fields pass through untouched; every OTHER odoo (numeric) field is rounded — -# mirrors aios-web/api/main.py _payload() exactly so embed == standalone byte-for-byte. - - -def _round(v): - return round(v) if isinstance(v, (int, float)) and not isinstance(v, bool) else v - - -def _round2(v): - """D-153 (W37, asked by lane B) — MONEY AND PERCENT KEEP THEIR DECIMALS. - - `_round`'s whole-number convention is right for counts and wrong for currency: `first_cost` - 3.47 shipped as `3`, so every money cell on every Odoo grid was a whole dollar. - - ⚠ `pct` IS AFFECTED TOO, and that is the half most likely to be dropped. `cells.ts` renders a - percent as `num(v).toFixed(1) + "%"`, so a `yoy_pct` of 12.34 was `round()`ed to 12 and painted - "12.0%" — one decimal of a number that HAS one, which reads as precision rather than as loss. - ⛔ NOT written as a `_round(v, nd=0)` default argument, deliberately: `round(v, 0)` returns a - FLOAT where `round(v)` returns an INT, so every integer column's wire shape would change under - an edit that looks purely cosmetic. - """ - return round(v, 2) if isinstance(v, (int, float)) and not isinstance(v, bool) else v - - -# Types a USER may create from the column menu (owner item 7, 2026-07-26). Mirrors -# customer-grid/types.ts CREATABLE_TYPES; verify_fields_contract.py holds the two in step. -# select — a single-select with its own `options` (the "Status" a user wants to add; distinct -# from the Odoo-sourced `status` lifecycle, which is not user-defined) -# user — an assignee, whose choices come from the HOST's real user list, never from here -#: `multiselect` (wave-2 item 5, 2026-07-27): the Airtable-style MULTI select — declared -#: options like `select`, but the cell holds a comma-joined SET and the row belongs to every -#: member of it (the `multi` grouping/cell contract the Cohorts column established). -#: Wave-5 item 11 (2026-07-27): `checkbox` (cell = bool; the overlay stores '1' or '') · -#: `phone` / `email` / `url` (text-family with per-type rendering) · `rating` (top-level -#: `max`, SVG stars client-side — never emoji) · `created_time` (READ-ONLY, renders the row's -#: `_created`) · `formula` (READ-ONLY, client-computed from the row's other cells). -#: ⭐ Wave-19 R7 / contract C5: `image` — a PICTURE on a record. The cell holds a string -#: REFERENCE, never bytes: a product `code`, an `ed:` editorial asset, or `rec:` for -#: something uploaded through the field (`POST /api/v1/assets/records`). Storing a reference is -#: what lets Royal's 1,142 existing SKU masters appear with nothing re-uploaded, and it keeps the -#: overlay stratum the size it is — base64 bytes in a JSON blob read on every render would be a -#: megabyte-per-row tax on the whole store. Editable by NATURE (deliberately NOT in -#: `READONLY_CUSTOM_TYPES`): the ref is what the upload endpoint hands back, and the client PATCHes -#: it through the ordinary overlay wall rather than the asset route writing cells behind it. -#: ⭐ WAVE 23 (C7) — `json` joined: a cell holding a whole DOCUMENT (an Instagram comment thread, -#: a webhook payload, a scraped blob) that opens in its own viewer instead of being flattened -#: into one unreadable line. It is EDITABLE by nature, like `image`: the value is still a plain -#: string on the wire, so it rides the ordinary overlay wall — what makes it a json field is that -#: `grid_events` REFUSES a write that does not parse (a column promising structure must not -#: silently hold something that isn't). -#: ⭐ 2026-08-07 — `link` and `rollup` JOINED (the relational wave). They ride here because -#: `UT_FIELD_TYPES` must stay a SUBSET of this set (gated in verify_api's W18-UT section) — the -#: surfaces that CREATE them are the user-table databases, where a relation between two tables is -#: a thing that exists. On the Odoo-backed Customer/Product grids there is no second user table to -#: point at, so the column menu there simply never offers one. -#: ⭐⭐ WAVE-34 (owner ruling R13) — `ai_enrich` JOINS, and it had to join HERE in the same change -#: that put it in `UT_FIELD_TYPES`, not a ticket later. The wave planned a SERVER-FIRST landing on -#: the reasoning that a kind the client does not offer is invisible while a kind the server refuses -#: deletes a column. That reasoning is sound and the conclusion was still wrong, because THREE -#: parity gates chain over these sets and none of them permits a partial landing: -#: `verify_api` W18-UT `UT_FIELD_TYPES - CUSTOM_FIELD_TYPES == set()` (this line) -#: `aw_fields_contract` §5 `CREATABLE_TYPES == CUSTOM_FIELD_TYPES`, EXACT set equality -#: `types.ts` `CREATABLE_TYPES: readonly FieldType[]`, so the union must carry it too -#: Measured live by lane B at 17:09: `api_api` went 969/969 to 968/969 the moment the kind entered -#: `core/user_tables.py` alone, printing `got {'ai_enrich'} want set()`. The comments on -#: `json`, `link`/`rollup` and `code` below all say the same thing in their own words. -CUSTOM_FIELD_TYPES = {"text", "select", "multiselect", "user", "int", "currency", "pct", "date", - "checkbox", "phone", "email", "url", "rating", "created_time", "formula", - "automation", "image", "json", "link", "rollup", "code", "ai_enrich"} - -#: ⭐ WAVE-27 item 13 (owner ruling R13) — the `code` field's LANGUAGES. -#: -#: R13 is explicit that this kind is "syntax-highlighted storage + language config, NO execution -#: engine". So a language is a RENDERING hint and nothing else: it selects a highlighter, it never -#: selects an interpreter, and no value here may ever grow a run path. The list is short on -#: purpose — every entry costs a highlighter the client actually has to implement, and an -#: unimplemented language would paint plain text under a label promising colour. -#: -#: `plain` is the default and the fallback, so it is never a second way to say nothing: it is the -#: honest answer for a snippet whose language the user has not chosen. -CODE_LANGUAGES = {"plain", "json", "sql", "python", "javascript", "typescript", - "html", "css", "markdown", "yaml", "xml", "shell"} - - -def _clean_code(raw): - """The `code` field's config bag -> `{'language': ...}`, or None. - - Deliberately OPTIONAL rather than required (the `automation` posture, not `link`'s): a code - column with no declared language is a legitimate state — it stores and highlights as plain - text — so refusing the FIELD over a missing bag would block the ordinary create path. An - unknown language falls back to `plain` rather than refusing, because the value is a rendering - hint: dropping the user's column to punish a typo in a highlighter name would be the - disproportionate half of the fail-closed rule. - - ⚠ `plain` RETURNS NONE, and that is what makes the control reversible rather than one-way. - Absent already means plain, so storing `{'language': 'plain'}` would be the default wearing a - second name (the `kanbanClamp` law). But the patch path resolves an OMITTED key to the - previous value — so if plain were merely omitted by the client, switching a column back from - SQL to Plain text would keep storing SQL and read as a control that does not save. Sending - the bag explicitly and having it evaluate to None here means: omit = keep, plain = clear. - """ - if not isinstance(raw, dict): - return None - lang = str(raw.get('language') or 'plain').strip().lower() - if lang not in CODE_LANGUAGES or lang == 'plain': - return None - return {'language': lang} -#: User-created types whose CELLS are read-only: their values are computed (formula — client -#: side, any error degrades to BLANK) or system-owned (created_time = the row's `_created`). -#: Emitted with the cohort column's read-only mechanism — `source: 'odoo'` + `derived` — so -#: the client never offers an editor and the host never accepts a cell write for them. -READONLY_CUSTOM_TYPES = {"created_time", "formula"} -MAX_FIELD_OPTIONS = 50 -MAX_FORMULA_LEN = 500 -#: `rating` bounds. Airtable caps at 10; below 2 a rating is a checkbox. -RATING_MAX_DEFAULT, RATING_MAX_MIN, RATING_MAX_MAX = 5, 2, 10 - -#: Key prefix of a FORMULA-MEASURE column (owner item 7, 2026-07-27): a user-created field that -#: IS a measure over a window — `Sales · the last 90 days` as a column. Mirrors the client's -#: `measure_` keys in CustomerGrid.createField. Distinct from `custom_` because the two strata -#: could not be more different: `custom_` is the EDITABLE overlay (user-typed values), while a -#: measure field is READ-ONLY and its values are computed by the host per render. -MEASURE_FIELD_PREFIX = "measure_" -#: The numeric types a measure can render as (semantic._FORMAT_TYPE's range). -MEASURE_FIELD_TYPES = {"currency", "int", "pct"} - - -def _clean_options(raw): - """Choices for a `select`: strings, trimmed, de-duplicated case-insensitively, capped. - Mirrors types.ts parseOptions — a choice list that means one thing in the picker and - another in the filter dropdown is a column with two vocabularies.""" - out, seen = [], set() - for v in list(raw or [])[:MAX_FIELD_OPTIONS * 2]: - if not isinstance(v, (str, int, float)) or isinstance(v, bool): - continue - s = str(v).strip()[:120] - if not s or s.lower() in seen: - continue - seen.add(s.lower()) - out.append(s) - return out[:MAX_FIELD_OPTIONS] - - -def _clean_option_colors(raw, options): - """Choice-label -> #RRGGBB, limited to the field's canonical option vocabulary.""" - if not isinstance(raw, dict): - return {} - supplied = {} - for label, color in raw.items(): - if not isinstance(label, str) or not isinstance(color, str): - continue - clean = color.strip().upper() - if re.fullmatch(r"#[0-9A-F]{6}", clean): - supplied[label.strip().lower()] = clean - out = {} - for option in options or []: - color = supplied.get(str(option).strip().lower()) - if color: - out[str(option)] = color - return out - - -def _choice_appearance(raw, options): - """Validated select-family appearance. Absent colour toggle means legacy-on.""" - if not isinstance(raw, dict): - return {} - out = {} - if isinstance(raw.get("colorCodeOptions"), bool): - out["colorCodeOptions"] = raw["colorCodeOptions"] - colors = _clean_option_colors(raw.get("optionColors"), options) - if colors: - out["optionColors"] = colors - return out - - -def _clean_rating_max(raw): - """A rating's star count, bounded. Anything unusable is the default, not a refusal — the - field still holds its 1..max integers either way.""" - try: - return max(RATING_MAX_MIN, min(int(raw), RATING_MAX_MAX)) - except (TypeError, ValueError): - return RATING_MAX_DEFAULT - - -#: The field types a number-style display format may apply to. `formula` is here because its -#: RESULT is a number the client renders; `pct` already renders in points and takes decimals. -_NUMBER_FORMAT_TYPES = {"int", "currency", "pct", "formula"} - - -def _clean_format(raw, ftype): - """Per-type DISPLAY format (wave-5 item 10), fail-closed: unknown keys are DROPPED, wrong - types return None (the property is simply absent). Rendering-only — a format can change how - a value reads, never what it is, which is why this needs no parity gate of its own.""" - if not isinstance(raw, dict): - return None - out = {} - if ftype in _NUMBER_FORMAT_TYPES: - if isinstance(raw.get("thousands"), bool): - out["thousands"] = raw["thousands"] - if raw.get("decimals") is not None: - try: - d = int(raw["decimals"]) - except (TypeError, ValueError): - d = None - if d is not None and 0 <= d <= 4: - out["decimals"] = d - if isinstance(raw.get("abbrev"), bool): - out["abbrev"] = raw["abbrev"] - elif ftype in ("date", "created_time"): - if isinstance(raw.get("time"), bool): - out["time"] = raw["time"] - if raw.get("tz") in ("local", "utc"): - out["tz"] = raw["tz"] - return out or None - - -def _clean_permissions(raw): - """`{edit: 'everyone' | 'creator' | 'admins'}` or None (wave-5 item 1). WHO may set it is - the host handler's business (creator/admin, enforced fail-closed there); this validates - only the shape, like every other property here.""" - if isinstance(raw, dict) and raw.get("edit") in ("everyone", "creator", "admins"): - return {"edit": raw["edit"]} - return None - - -#: ⛔ THE CHARSET MUST ADMIT EVERY TOKEN THE CLIENT ENGINE PARSES, or a legal formula is -#: refused by a wall that is supposed to be structural (2026-08-03). -#: -#: This regex was written when a formula was arithmetic over refs. On 2026-07-31 the client -#: engine (owner item 2) gained STRING literals, `&` concatenation and `^` — CONCATENATE, TEXT, -#: LEFT/RIGHT/MID, and any `IF(cond, "yes", "no")`. This list was never widened to match, so -#: every such formula died here: `field_upsert` refused the create, and `fields_from_workspace` -#: dropped the column on read. Nothing went red — a refused create looks like a quiet failure -#: and a dropped column looks like a column nobody made. -#: -#: Found by trying to ship the owner's own Buy signal formula, which is `IF(..., "Buy now", -#: "OK")` and could not be created through the product UI at all. -#: -#: ⚠ IT IS STILL STRUCTURAL, and deliberately not a second grammar — that is the filter_sql-class -#: drift risk this file's docstring names. The charset says which characters may appear; the -#: engine says what they mean. `_quotes_balanced` below is the one structural rule the quote -#: character brings with it. -_FORMULA_CHARS = re.compile(r"^[\w\s{}()+\-*/.,<>=!^&\"]*$") -_FORMULA_REF = re.compile(r"\{([^{}]*)\}") - - -def _quotes_balanced(s): - """An even number of `"` — the structural half of string support. - - Sound because the engine's own escape is Excel's: `""` inside a string is one quote, and it - contributes TWO characters. So a well-formed expression always has an even count and an - unterminated string always has an odd one. What a balanced pair MEANS is the engine's - business, exactly as with parentheses. - """ - return s.count('"') % 2 == 0 - - -#: Wave-18 C5-AUTOFIELD. `kind` is a whitelist because an unknown kind would be a column that -#: silently never runs; `source` names where the run's subject comes from. -AUTOMATION_KINDS = {"instagram_profile"} -AUTOMATION_SOURCES = {"record_url_field", "self"} -MAX_AUTOMATION_SETTINGS = 12 - - -def _clean_automation(raw, valid_keys=None, flow_ids=None): - """Validate an `automation` config bag. Returns the clean dict, or None when there is - nothing valid to store (the column then renders as unconfigured — never invented). - - ⛔ WAVE 22 (contract C8, owner item 5) — NO FIELD WITHOUT A FLOW. With `flow_ids` given - (the WRITE path: grid_events / user_tables pass the tenant's automation-definition ids), - the bag MUST carry a `flowId` naming one of them — absent or naming a deleted flow is - refused, the same fail-closed direction as a formula ref that names no field. With - `flow_ids=None` (the READ path, `fields_from_workspace`) the law is NOT applied: a column - stored before the law must keep projecting — enforcement at read time would vaporise it, - which is the `_clean_formula` write/read split exactly. - """ - if not isinstance(raw, dict): - return None - kind = str(raw.get('kind') or '').strip() - if kind not in AUTOMATION_KINDS: - return None - source = str(raw.get('source') or 'record_url_field').strip() - if source not in AUTOMATION_SOURCES: - source = 'record_url_field' - out = {'kind': kind, 'source': source} - flow = str(raw.get('flowId') or '').strip()[:40] - if flow_ids is not None and (not flow or flow not in flow_ids): - return None - if flow: - out['flowId'] = flow - url_field = str(raw.get('urlField') or '').strip()[:80] - # fail closed on a ref that does not exist, exactly as _clean_formula does at WRITE time - if url_field and (valid_keys is None or url_field in valid_keys): - out['urlField'] = url_field - settings = {} - for k, v in list((raw.get('settings') or {}).items())[:MAX_AUTOMATION_SETTINGS]: - if isinstance(v, bool) or isinstance(v, (int, float)): - settings[str(k)[:40]] = v - elif isinstance(v, str): - settings[str(k)[:40]] = v[:200] - if settings: - out['settings'] = settings - return out - - -def _clean_formula(raw, valid_keys=None): - """STRUCTURAL passthrough for a formula field's expression (wave-5 item 9). - - Meaning is NOT checked here: the CLIENT engine owns the grammar (arithmetic over `{field}` - refs, ABS/ROUND/MIN/MAX/IF), and any evaluation error degrades to a BLANK cell — never a - wrong number. That is the `_clean_window` split one stratum up, and deliberately NOT a - Python mirror of the grammar: a second engine is the filter_sql-class drift risk. - Structure IS checked — charset, length, balanced parens, BALANCED QUOTES, well-formed - non-empty `{refs}` — and at WRITE time (`valid_keys` given) every ref must name a field this - table has, fail closed. At READ time refs are left alone: a referenced field deleted later - must blank the CELLS, not vaporise the column. - - ⚠ A `{ref}` INSIDE A STRING LITERAL is still checked against `valid_keys` at write time, so - `IF(x, "see {notafield}", "")` is refused. That is a false rejection and it is the - fail-closed direction: the alternative is teaching this function where strings begin and - end, which is the second grammar the paragraph above refuses to write. - """ - if not isinstance(raw, str): - return None - s = raw.strip() - if not s or len(s) > MAX_FORMULA_LEN or not _FORMULA_CHARS.match(s): - return None - depth = 0 - for ch in s: - if ch == "(": - depth += 1 - elif ch == ")": - depth -= 1 - if depth < 0: - return None - if depth: - return None - if not _quotes_balanced(s): - return None - refs = _FORMULA_REF.findall(s) - leftover = _FORMULA_REF.sub("", s) - if "{" in leftover or "}" in leftover: # unbalanced / nested braces - return None - if any(not r.strip() for r in refs): # a `{}` ref names nothing - return None - if valid_keys is not None and any(r not in valid_keys for r in refs): - return None - return s - - -def _clean_geocode(raw): - """One `geocode` bag → the stored shape, or None (dropped). - - ⭐⭐ W37-T22 (owner item 15 / ruling R4, contract C7). THE ONE validator for the geocode - pseudo-kind, called by BOTH field doors: `_field_extras` here (the Odoo grids) and - `core/user_tables._clean_field` via `_ag_geocode` (every `ut_*` user database). - - ⛔ IT WAS INLINE IN `_field_extras` UNTIL 2026-08-19 AND THAT IS WHY IT SHIPPED HALF-DONE. - `user_tables._clean_field` is a strict allowlist that returns only the bags it NAMES, so a - geocode column created on a user database was stored WITHOUT its `{addressField}` — the - *"created, named, configured, gone"* failure the `image` type shipped as in wave 19. Extracting - it rather than copying the rules is [[one-question-two-normalizers]]: a second copy is how the - two doors come to disagree about what a valid bag is. - - ⛔ WHOLE-KEY DROP (the `swipe` construction): a geocode column with no address column is not - a degraded column, it is a column no run can ever fill, and the honest state for that is the - unconfigured one the client refuses at `canCreate`. - - ⛔ THE FIELD'S `type` IS `text` AND THERE IS NO `geocode` FieldType. The kind is a PSEUDO-KIND - in the create picker (the `measure` construction), so `CUSTOM_FIELD_TYPES`, `UT_FIELD_TYPES`, - `CREATABLE_TYPES`, the shape/label tables and the cell renderer are all untouched — three - parity gates chain over those sets and none permits a partial landing. This is also why the - bag takes `format`'s posture and NOT `formula`'s: there is no type to pair it with, so an - absent bag is an ordinary text column rather than a refusal. - - ⚠ `addressField` is NOT checked against the table's keys: neither door is given `valid_keys`, - and the split this module already draws puts that loss on the surface that RENDERS. The - enricher must SAY the address column is gone rather than geocode an empty string. - """ - if not isinstance(raw, dict): - return None - addr = raw.get("addressField") - if not isinstance(addr, str) or not addr.strip(): - return None - out = {"addressField": addr.strip()[:80]} - # D-3 — the optional ISO 3166-1 alpha-2 hint, mapped straight onto the geocoder's - # `countrycodes` parameter. NORMALISED to upper case and stored only when it is exactly two - # letters: absent means "anywhere", and `""` would be that state's second spelling. A - # three-letter or numeric code is REFUSED rather than truncated, because a truncated code is - # a valid code for a different country. - cc = raw.get("country") - if isinstance(cc, str) and re.fullmatch(r"[A-Za-z]{2}", cc.strip() or ""): - out["country"] = cc.strip().upper() - return out - - -def _field_extras(saved, ftype): - """createdBy / permissions / format / scope — the validated passthrough the created strata - share (wave 5). `createdBy` is only ever WRITTEN host-side (the handler stamps it); here it - is carried so the client can gate its menus and the handler can enforce against it. - `scope` (wave-6 item 9): 'cohort' marks a field created as cohort-specific — the handler - stamps it at create (cohort page only) and preserves it like createdBy; carried here so the - client can label the field, filtered OUT of other pages by fields_from_workspace.""" - out = {} - who = saved.get("createdBy") - if isinstance(who, str) and who.strip(): - out["createdBy"] = who.strip()[:80] - perms = _clean_permissions(saved.get("permissions")) - if perms: - out["permissions"] = perms - fmt = _clean_format(saved.get("format"), ftype) - if fmt: - out["format"] = fmt - if saved.get("scope") == "cohort": - out["scope"] = "cohort" - # ── ⭐⭐ W37-T22 (owner item 15 / ruling R4, contract C7): THE GEOCODE COLUMN'S CONFIG ── - # - # `{addressField}` — which column this one reads. WHOLE-KEY drop (the `swipe` construction): - # a geocode column with no address column is not a degraded one, it is a column no run can - # ever fill, and the honest state for that is the unconfigured one the client refuses at - # `canCreate`. - # - # ⛔ THE FIELD'S `type` IS `text` AND THERE IS NO `geocode` FieldType. The kind is a PSEUDO-KIND - # in the create picker (the `measure` construction), so `CUSTOM_FIELD_TYPES`, `UT_FIELD_TYPES`, - # `CREATABLE_TYPES`, the shape/label tables and the cell renderer are all untouched — three - # parity gates chain over those sets and none permits a partial landing. - # ⚠ `addressField` is NOT checked against this table's keys: `_field_extras` is not given - # `valid_keys`, and the split this module already draws puts that loss on the surface that - # RENDERS ("what each mode MEANS ... is the client's business"). The enricher must SAY the - # address column is gone rather than geocode an empty string. - # ⚠ EXTRACTED to `_clean_geocode` 2026-08-19 (/validate-wave, W37-T22 clause 1). The rules - # did not change; what changed is that `core/user_tables.py` now reaches THIS function through - # `_ag_geocode` instead of carrying a second copy. See `_clean_geocode`'s own note. - geo = _clean_geocode(saved.get("geocode")) - if geo: - out["geocode"] = geo - corrected_from = saved.get("labelCorrectedFrom") - correction_id = saved.get("labelCorrectionId") - if (isinstance(corrected_from, str) and corrected_from.strip() - and isinstance(correction_id, str) and correction_id.strip()): - out["labelCorrectedFrom"] = corrected_from.strip()[:120] - out["labelCorrectionId"] = correction_id.strip()[:180] - return out - - -#: The DERIVED column listing the cohorts a customer is in (owner, 2026-07-27). -#: -#: NOT in `aios_grid_fields.json`, deliberately. That contract is per-TABLE and shared with the -#: standalone API and the dev sample; a cohort is per-USER, so the column exists exactly when the -#: caller has cohorts — the same condition under which the `__cohort__` FILTER field is offered. -#: Putting it in the canonical contract would mean an always-present column that is empty for -#: everyone else, plus three consumers to keep in step for a value none of them can produce. -COHORT_COLUMN = 'cohorts' - - -def cohort_field(label='Locked views'): - """The derived membership column's descriptor. - - ⚠ WAVE 17 item 14 / C-STR — THE LABEL AND THE NOTE SPEAK THE NEW VOCABULARY; THE KEY DOES - NOT. `COHORT_COLUMN` is still `'cohorts'` and the function is still `cohort_field`, because - every stored view that shows or groups by this column names it by KEY, and every gate in - two runtimes names the function. The owner renamed a CONCEPT ("we should stop calling it - Cohort, but locked instead"), which is a change to what a reader sees — renaming the - identifiers would break saved views to change a word nobody reads. - - `source: 'odoo'` is doing ONE job here and it is not provenance: the client keys editability - off `source == 'overlay'`, and `_cl_handle_grid_event` accepts cell writes only for overlay - keys. So 'odoo' is what makes this column READ-ONLY at both ends. `derived: True` is what - stops the column menu calling it a "source field" on that basis. - - ⚠ `filterable: False`, and the replacement is the `Where [Cohort] […]` CONDITION, not another - column. Text ops over a joined string would ALMOST work and disagree at the edges — `contains - "VIP"` also matches a cohort called "VIP club" — and a filter that is nearly right is worse - than one that is absent. - """ - return { - 'key': COHORT_COLUMN, 'label': label, 'type': 'text', 'source': 'odoo', - 'default': False, 'filterable': False, 'derived': True, 'multi': True, - 'note': 'The locked views this customer is in, newest first. A locked view holds a SET, ' - 'so grouping by this column puts a customer under EVERY view they are in — the ' - 'group counts therefore add up to more than the record count, which stays the ' - 'number of distinct customers. Read-only: membership changes only by adding or ' - 'removing customers on the locked view itself.', - } - - -def cohort_cells(cohorts, allowed_pids=None): - """`{pid: {'cohorts': 'Q3 calls, Lost'}}` from `[{id,name,pids}, ...]`. - - Built per render from the caller's OWN cohorts and handed to `rows_from_pool` as `derived`, - never merged into the cached pool rows — those are shared across users, and stamping one - user's cohorts onto them would leak the membership to everybody else on the next render. - """ - cells = {} - for c in cohorts or []: - # ⚠ The COMMA is the separator the client splits on to group a customer into EVERY - # cohort they are in, so it cannot also occur inside a name. Cohort names are free text - # ("Q3 calls, west" is a name somebody will type), so a comma is replaced here rather - # than left to break the split silently — one group called "Q3 calls" and another called - # "west" would be two lists that do not exist. The cost is cosmetic and confined to the - # cell; the Cohort page still shows the name the user typed. - name = str(c.get('name') or c.get('id') or '').replace(',', ' ').strip() - for pid in c.get('pids') or (): - if allowed_pids is not None and pid not in allowed_pids: - continue - cells.setdefault(pid, []).append(name) - return {pid: {COHORT_COLUMN: ', '.join(names)} for pid, names in cells.items()} - - -def clean_measure_field(raw, offered): - """Validate one UNTRUSTED formula-measure field (owner item 7) against the caller's OFFER. - - `offered` is `{measure key: {label, type}}` from `measure_filter.measure_fields(team_id)` — - the same admission the condition builder uses, so a field can only name a measure this - caller could also filter by. Returns the canonical stored shape, or None (fail closed). - - `source:'odoo'` + `derived:True` is the cohort column's read-only mechanism, reused: - the client keys editability off `source == 'overlay'` and the host accepts cell writes only - for overlay keys, so a measure column cannot be typed into at either end. `filterable:False` - because the REPLACEMENT is the measure CONDITION with the same measure and window — the - governed, gate-proved path (CG-8/CG-12) — not text ops over a derived cell. - """ - if not isinstance(raw, dict): - return None - key = str(raw.get("key") or "") - if not key.startswith(MEASURE_FIELD_PREFIX) or len(key) > 80: - return None - spec = raw.get("measure") - if not isinstance(spec, dict): - return None - m = (offered or {}).get(spec.get("key")) - if not m: - return None # not admitted for this caller -> fail closed - window = _clean_window(spec.get("window")) - if window is None: - return None # a measure column with no period is not a column - mtype = m.get("type") if m.get("type") in MEASURE_FIELD_TYPES else "currency" - return { - "key": key, - "label": str(raw.get("label") or m.get("label") or "Measure")[:120], - "type": mtype, - "source": "odoo", - "default": True, - "custom": True, - "derived": True, - "filterable": False, - "agg": "sum" if mtype in ("currency", "int") else None, - "note": str(raw.get("note") or "")[:2000], - "measure": {"key": str(spec.get("key"))[:80], "window": window}, - } - - -def measure_fields_of(fields): - """The formula-measure columns among `fields` — the ones whose values the host must compute - per render (see `rows_from_pool`'s `derived`).""" - return [f for f in fields or [] if isinstance(f.get("measure"), dict)] - - -def fields_from_workspace(workspace=None, cohorts=False, scope_key=None, fields_base=None): - """Overlay persisted notes/custom fields onto the immutable source-field contract. - - `cohorts=True` appends the derived cohort column — see `cohort_field`. Appended LAST and - `default: False`, so it never displaces a column somebody already reads; the Fields menu is - where you turn it on. - - Three saved strata pass through: notes on base fields, `custom_` overlay fields (editable), - and `measure_` formula-measure fields (read-only, host-computed — owner item 7). A measure - field was validated against the caller's measure OFFER when it was written - (`clean_measure_field`); here only its SHAPE is re-checked, because this module has zero - app-internal imports and cannot know the offer. A measure that has since become - unanswerable (a BU scope on a company-level measure) degrades to a BLANK column at value - time, never to an error. - - `scope_key` (wave-6 item 9) names the PAGE doing the asking ('customer' / 'cohort'). A - saved def carrying `scope` is emitted only when it matches — so a cohort-specific field - never appears on the Customer table. FAIL-CLOSED: a caller that passes no scope_key sees - only unscoped (global) fields; base contract fields are never scoped. - - `fields_base` (wave 16 C-TOPIC) — the canonical contract to overlay onto. Absent = the - CUSTOMER contract (`FIELDS`), byte-identical to before the parameter existed; the product - surface passes `product_fields()`. The workspace dict a caller hands in must already be the - matching table object's bucket — this function cannot tell a customer overlay from a - product one, which is exactly why the buckets are separate stores. - """ - saved = dict((workspace or {}).get("fields") or {}) - out = [] - base_fields = fields_base if fields_base is not None else FIELDS - base_keys = {field["key"] for field in base_fields} - for base in base_fields: - meta = saved.get(base["key"]) or {} - field = dict(base) - if isinstance(meta.get("note"), str): - field["note"] = meta["note"][:2000] - # Wave-5 item 10: a saved DISPLAY format on any base field (a preset included) — how a - # number or date READS, per user. Rendering-only, so this is the whole acceptance. - fmt = _clean_format(meta.get("format"), base.get("type")) - if fmt: - field["format"] = fmt - # ⭐ W29-T83 — the saved COLUMN SUMMARY, the read half of the write door in - # `grid_events.field_upsert`. Without this the value round-trips into the store and is - # never served back, which looks exactly like a write that never happened - # ([[read-path-cannot-witness-write-path]]). Absent = whatever the contract declares. - if meta.get("agg") in FIELD_AGGS: - field["agg"] = meta["agg"] - # PRESET measure fields (wave-2 item 8): a preset+measure base may take a saved - # window/label override. Wave 6 deleted every preset member (the owner's - # no-buildable-presets rule) so this branch is currently MEMBERLESS — kept as the - # measure_ path's twin for any future preset-carrying contract, and because deleting - # it would silently change what a re-added preset means. - if base.get("preset") and isinstance(base.get("measure"), dict): - saved_measure = meta.get("measure") if isinstance(meta.get("measure"), dict) else {} - window = _clean_window(saved_measure.get("window")) - if window is not None: - field["measure"] = {"key": base["measure"]["key"], "window": window} - if isinstance(meta.get("label"), str) and meta["label"].strip(): - field["label"] = meta["label"][:120] - out.append(field) - for key, field in saved.items(): - if key in base_keys or not isinstance(field, dict): - continue - if field.get("scope") and field.get("scope") != scope_key: - continue # a cohort-specific field on another page (wave-6 item 9) - if (str(key).startswith(MEASURE_FIELD_PREFIX) - and isinstance(field.get("measure"), dict)): - window = _clean_window(field["measure"].get("window")) - mkey = str(field["measure"].get("key") or "") - if window is None or not mkey: - continue - mtype = field.get("type") if field.get("type") in MEASURE_FIELD_TYPES else "currency" - out.append({ - "key": str(key)[:80], - "label": str(field.get("label") or "Measure")[:120], - "type": mtype, - "source": "odoo", - "default": bool(field.get("default", True)), - "custom": True, - "derived": True, - "filterable": False, - "agg": "sum" if mtype in ("currency", "int") else None, - "note": str(field.get("note") or "")[:2000], - "measure": {"key": mkey[:80], "window": window}, - **_field_extras(field, mtype), - }) - continue - if not str(key).startswith("custom_"): - continue - ftype = field.get("type") - if ftype not in CUSTOM_FIELD_TYPES: - continue - if ftype in READONLY_CUSTOM_TYPES: - # Wave-5 items 9/11: the read-only user-created pair. Emitted with the cohort - # column's mechanism (source 'odoo' + derived) so the client never offers an - # editor and the host's overlay-write guard excludes them by construction. - # FILTERABLE since wave 6 (owner item 6): their values live client-side - # (formula computes over the row, created_time renders `_created`), this - # table's counts are client-mode, and the windowed count path never sees - # these tables — so the client engine answers them soundly. - entry = { - "key": str(key)[:80], - "label": str(field.get("label") or "Untitled")[:120], - "type": ftype, - "source": "odoo", - "derived": True, - "filterable": True, - "default": bool(field.get("default", True)), - "custom": True, - "note": str(field.get("note") or "")[:2000], - **_field_extras(field, ftype), - } - if ftype == "formula": - formula = _clean_formula(field.get("formula")) - if formula is None: - continue # a formula field without a formula is nothing - entry["formula"] = formula - out.append(entry) - continue - if field.get("source") != "overlay": - continue - options = (_clean_options(field.get("options")) - if ftype in ("select", "multiselect") else []) - if ftype in ("select", "multiselect") and not options: - # A select with no surviving choices can never hold a value. Dropping the COLUMN - # would lose the user's data; degrading it to text keeps every stored value - # readable and lets them re-add choices. - ftype = "text" - out.append({ - "key": str(key)[:80], - "label": str(field.get("label") or "Untitled")[:120], - "type": ftype, - "source": "overlay", - "default": bool(field.get("default", True)), - "custom": True, - # WAVE-29 C7: the whole vocabulary, not the `{"sum"}` literal that was here — a - # picker offering Average against a projection that only passes Sum through is the - # silent half of this feature. - "agg": field.get("agg") if field.get("agg") in FIELD_AGGS else None, - "note": str(field.get("note") or "")[:2000], - **({"options": options} if ftype in ("select", "multiselect") else {}), - **(_choice_appearance(field, options) - if ftype in ("select", "multiselect") else {}), - # comma-joined SET semantics (the Cohorts column's contract): the row belongs to - # every member, groups count it under each, the toolbar count stays distinct. - **({"multi": True} if ftype == "multiselect" else {}), - **({"max": _clean_rating_max(field.get("max"))} if ftype == "rating" else {}), - **({"automation": _clean_automation(field.get("automation"))} - if ftype == "automation" and _clean_automation(field.get("automation")) else {}), - # ⭐ WAVE-27 item 13 (R13) — the code column's language rides the wire, because the - # highlighter is chosen per column and the client cannot infer a language from a - # string. Absent = `plain`, which is what an unconfigured code column renders as. - **({"code": _clean_code(field.get("code"))} - if ftype == "code" and _clean_code(field.get("code")) else {}), - **_field_extras(field, ftype), - }) - if cohorts and not any(f.get('key') == COHORT_COLUMN for f in out): - # ⚠ The emptiness check is WAVE 19's, and it is about the topics R9 opened this column to. - # A user table's field keys are slugged from whatever its creator typed, so a column - # literally called "Cohorts" produces the key `cohorts` — and appending here unguarded - # would put TWO fields with one key on the wire. The client indexes fields by key, so the - # duplicate does not error: it silently paints one column's values under the other's - # header. The user's own column wins; the derived one steps aside rather than shadowing it. - out.append(cohort_field()) - return out - - -def rows_from_pool(pool_rows, fields=None, overlays=None, derived=None): - """Map customer_data.pool() dicts -> the API row shape the grid expects: - pid + each Odoo field (numeric fields rounded, text/date passed through) + the - persisted external overlay. Mirrors the standalone API payload contract. - - `derived` is `{pid: {key: value}}` for columns the HOST computes per render rather than - reads off the pool row — today just the cohort column. A separate argument from `overlays` - on purpose: `overlays` is the PERSISTED user stratum, and putting a value there that is - never written back would make the dict mean two things. - """ - fields = fields or FIELDS - odoo_fields = [field for field in fields if field["source"] == "odoo"] - overlay_fields = [field for field in fields if field["source"] == "overlay"] - derived_keys = [field["key"] for field in fields if field.get("derived")] - overlays = overlays or {} - derived = derived or {} - out = [] - for r in pool_rows: - pid = r.get("pid") - # `_created` (wave-5 item 11) rides every row like `pid` does — the datum the - # `created_time` field type renders, regardless of that field's own key. Not a Field: - # it has no column of its own until a user creates one. `lat`/`lon` (wave-7 W11) ride - # the same way: the Map VIEW's data, nullable, deliberately not a column. - row = {"pid": pid, "_created": r.get("_created") or "", - "lat": r.get("lat"), "lon": r.get("lon")} - for field in odoo_fields: - k = field["key"] - if field.get("derived"): - continue # not on the pool row — filled from `derived` below - v = r.get(k) - # D-153 — three roundings, not two: text/status/date pass through verbatim, currency - # and pct keep two decimals (`_round2`), everything else stays a whole number. - row[k] = v if field["type"] in {"text", "status", "date"} else ( - _round2(v) if field["type"] in {"currency", "pct"} else _round(v)) - saved = overlays.get(str(pid), {}) or {} - for field in overlay_fields: - row[field["key"]] = saved.get(field["key"], "") - got = derived.get(pid) or {} - for k in derived_keys: - # '' not None: a customer in no cohort has an EMPTY cohort list, and `is empty` on a - # text column is the question somebody will ask of it. - row[k] = got.get(k, "") - out.append(row) - return out - - -# --- the FILTER-TREE contract (mirrors customer-grid/types.ts) --------------- -# Ops the Airtable-parity condition builder can emit. isEmpty/isNotEmpty are -# VALUE-FREE (they legitimately carry no value and must never be dropped for it). -FILTER_OPS = {'contains', 'doesNotContain', 'eq', 'neq', 'isEmpty', 'isNotEmpty', - 'gt', 'gte', 'lt', 'lte', 'between', 'within', - # Wave 2026-08-02 (C-OPS): RANK operators — evaluated as a SET pass over the - # sibling-filtered domain by the client engine (useVisibleRows). The validator - # accepts them like any op (structural, not semantic); filter_sql REFUSES to - # compile them to row SQL (a per-row WHERE cannot express Top-N). aboveAvg / - # belowAvg are VALUE-FREE; the rest encode their argument in `value` as a - # string int (topN/bottomN 1..10000, inTopPct/inBottomPct 1..100, - # inQuartile 1..4, inDecile 1..10). Deliberately NOT in MEASURE_OPS: on a - # measure-carrying column they rank the field's own derived values. - 'topN', 'bottomN', 'inTopPct', 'inBottomPct', - 'aboveAvg', 'belowAvg', 'inQuartile', 'inDecile'} -#: The RESERVED pseudo-column of a cohort-membership leaf (owner item 5, 2026-07-26). It is not -#: a Field and never will be — a cohort is a hand-curated SET, so making it a column would mean -#: a cell per row per cohort. Mirrors customer-grid/types.ts COHORT_FIELD. -COHORT_FIELD = '__cohort__' -#: Ops a cohort leaf may carry (owner, 2026-07-27): set operators over a SET of cohorts. -#: Anything else is dropped. Deliberately DISJOINT from FILTER_OPS — see types.ts COHORT_OPS: -#: a set op reaching a column leaf would fall through the client engine's switch to "no -#: narrowing", so keeping the vocabularies apart makes the existing fail-closed drop do the work. -COHORT_OPS = {'anyOf', 'allOf', 'noneOf'} - -#: ⭐⭐ WAVE-37 T26 (owner item 14, contract C4) — THE VIEW-MEMBERSHIP LEAF's reserved -#: pseudo-column and its operators. Mirrors `customer-grid/types.ts` VIEW_FIELD / VIEW_OPS. -#: -#: ⛔ DISJOINT FROM `FILTER_OPS`, for the reason COHORT_OPS is: `matchFilter` answers an operator -#: it does not know by NOT NARROWING, so an op that reached a column leaf would widen the result -#: under an authoritative count. Keeping the two vocabularies apart means a column leaf carrying -#: `inView` is dropped by the same fail-closed path that drops `dropTable`. -#: ⚠ The leaf names exactly ONE view (singular where a cohort leaf names a set), so there is no -#: all-or-nothing rule to get wrong here — the id either resolves or the leaf goes. -VIEW_FIELD = '__view__' -VIEW_OPS = {'inView', 'notInView'} -MAX_VIEW_ID = 64 -#: The single-cohort ops this leaf shipped with, kept as PERMANENT aliases and REWRITTEN here: -#: `is part of [one]` is `is any of [that one]`, so a saved view keeps answering and upgrades the -#: next time it is written. Mirrors types.ts COHORT_OP_ALIASES. -COHORT_OP_ALIASES = {'eq': 'anyOf', 'neq': 'noneOf'} -#: How many cohorts one condition may name. Mirrors types.ts MAX_COHORT_IDS. -MAX_COHORT_IDS = 20 - - -def parse_cohort_ids(value): - """The cohorts a leaf names, parsed out of `value`. Mirrors types.ts `cohortIds()`. - - Comma-separated in one string because `FilterRule.value` is what all four layers persist and - round-trip, and a one-element list is byte-identical to what the single-cohort leaf already - stored — so every shipped view parses with no migration. Safe because a cohort id is built - from `[a-z0-9_]` only (modules/cohort.new_id), so a comma cannot occur inside one. - """ - out = [] - for raw in ('' if value is None else str(value)).split(','): - cid = raw.strip()[:120] - if not cid or cid in out: - continue - out.append(cid) - if len(out) >= MAX_COHORT_IDS: - break - return out -# Airtable allows 3 nesting levels (root conditions -> group -> group), then grays -# the button out. MAX_FILTER_DEPTH in types.ts must stay in lock-step with this. -MAX_FILTER_DEPTH = 3 -MAX_FILTER_NODES = 100 # total nodes across the whole tree -MAX_FILTER_SIBLINGS = 50 # per level - - -#: Shape of a measure condition's date window. aios_grid has ZERO app-internal imports by -#: design, so it does NOT know the window VOCABULARY — `harness/windows.py` owns that, mirrored -#: in `customer-grid/windows.ts`, and a third copy here is exactly the drift those two already -#: need a gate to prevent. This validates SHAPE only. -WINDOW_MAX_N = 3650 - - -def _clean_window(raw): - """Structural passthrough for a measure condition's `{kind, n?, from?, to?}` window. - - Meaning is NOT checked here: an unrecognised `kind` survives this function and is REFUSED by - `harness.measure_filter.resolve_rule`, the layer that owns the vocabulary. Splitting it this - way keeps the grid module reusable and keeps one definition of what "last quarter" means. - """ - if not isinstance(raw, dict): - return None - kind = raw.get('kind') - if not isinstance(kind, str) or not kind or len(kind) > 40: - return None - out = {'kind': kind} - if raw.get('n') is not None: - try: - out['n'] = max(1, min(int(raw['n']), WINDOW_MAX_N)) - except (TypeError, ValueError): - return None - for side in ('from', 'to'): - if raw.get(side) not in (None, ''): - out[side] = str(raw[side])[:32] - return out - - -def _clean_rhs(raw, valid_keys): - """CG-9 — validate `{kind, colId, window?}`, the "compare against another attribute" side. - - SHAPE and KEY only: `colId` must be something this table has (the caller widens `valid_keys` - with the measure keys, exactly as it does for the left side), and a measure rhs must carry a - window. What the window MEANS is `harness/windows.py`'s business, same split as `_clean_window`. - """ - if not isinstance(raw, dict): - return None - kind = raw.get('kind') - if kind not in ('field', 'measure', 'stat'): - return None - if kind == 'stat': - # A STATISTIC carries no column: the population is the comparand. Shape only — which - # statistics exist is `harness/measure_filter.STATS`'s business, and an unrecognised one - # is REFUSED there rather than guessed at, exactly like an unrecognised window kind. - stat = raw.get('stat') - if not isinstance(stat, str) or not stat or len(stat) > 24: - return None - return {'kind': 'stat', 'stat': stat} - col = raw.get('colId') - if col not in valid_keys: - return None - out = {'kind': kind, 'colId': col} - if kind == 'measure': - window = _clean_window(raw.get('window')) - if window is None: - return None # a measure comparand with no period is not a question - out['window'] = window - return out - - -#: View DISPLAY MODES beside the grid (wave-6 item 10; 'map' wave-7 W11; 'dashboard' wave-8 -#: I19). Mirrors customer-grid/types.ts DISPLAY_MODES; 'grid' is what an absent/unknown -#: display means, so it is never stored. -#: ⚠ 'dashboard' is RETAINED FOREVER (wave-9 I10, contract C2). The owner renamed the mode to -#: "Chart" (a Dashboard MODULE is coming and the two would collide), but this set is the -#: gatekeeper for a STORED value: `_clean_display` DROPS an unknown mode, so removing -#: 'dashboard' here would silently downgrade every already-saved chart view to grid — and live -#: views are sitting in mode:'dashboard' right now (wave 8's own close-out records one). The -#: rename is therefore a stored-value MIGRATION, not a constant rename: accept 'dashboard' on -#: READ forever, only ever WRITE 'chart'. -#: ⭐ WAVE-27 item 8 (owner ruling R2, contract C3): 'swipe' — a DECK of the records whose bound -#: single-select is EMPTY, triaged one at a time by swiping left or right into two of that -#: field's options. Landed here FIRST and in the same change as the client registry, which is -#: the whole reason the two modes above it needed a staged hold: `_clean_display` DROPS an -#: unknown mode, so a client that offers a mode this set does not carry lets a user build a view -#: that silently reverts to a grid on the next read. -#: ⭐⭐ WAVE-29 R6/R7 (owner item 10, contract C3) — 'form', which CLOSES D-90. The client has -#: carried `form` in its union, with an icon, a label and a tone, since wave 23; this set never -#: did, so `_clean_display` DROPPED both the mode and the `display.form` spec on every write — -#: while `routes_forms.py` reads exactly that key to serve the public submit door. The public door -#: has therefore been live and UNREACHABLE for two waves: not broken, just impossible to point at -#: anything. The mirror is one name, and it is the half nobody could see was missing because -#: BOTH sides were individually consistent. -#: ⚠ Being a legal stored mode is NOT the same as being offered: `form` is deliberately held out -#: of the client's `CREATABLE_MODES` until `CustomerGrid` mounts a renderer for it (the hold law -#: written into `iconShapes.ts`, and now machine-enforced in BOTH directions by -#: `verify_icons.py::mode_parity` — offering an unmounted mode is red, and mounting an unoffered -#: one is red too, so the hold cannot outlive its reason the way wave 27's did). -#: ⭐⭐ WAVE-37 T20 (owner item 10, ruling R2, contract C2) — 'script', and the reason it lands -#: here is NOT the one the PRD gives. The PRD says the Custom View is invisible because this set -#: omits the name so `_clean_display` drops the mode on every write. MEASURED, that write never -#: happens: a script view is a rail PROJECTION built client-side by -#: `CustomerGrid.tsx::scriptProjectionView`, its panel is gated on `activeScriptId !== null` -#: (a row in E's per-database store), there is no `displayMode === "script"` branch at all, and the -#: autosave effect reads `views.find(...)` while the projections are appended to `railViews` ALONE. -#: Nothing was ever dropped, because nothing was ever sent. -#: ⭐ WHAT THE NAME ACTUALLY BUYS is the PRECONDITION for the create door the owner asked for: a -#: script view can only be offered through the display-mode picker once a view STORED as -#: `mode: 'script'` survives a round trip, and until this line it could not. The picker entry -#: itself is W37-T41 (lane E) plus `CREATABLE_MODES`, and `_test/icons.test.ts::HELD_MODES` keeps -#: the hold with its release condition rewritten to say so. -#: ⚠ CONTRACT C2 MAKES THIS SET AND `customer-grid/types.ts::DISPLAY_MODES` IDENTICAL NAME FOR -#: NAME, and `verify_fields_contract.py` now asserts it in BOTH directions. That retires the staged -#: hold every mode from `timeseries` to `form` was landed with — a client-first name is now RED -#: rather than merely risky. The protection the hold gave is structural instead: both halves are in -#: one lane's fence, so they ship in one change, and whether a mode may be CREATED is a separate -#: question answered by `CREATABLE_MODES` / `HELD_MODES`. -DISPLAY_MODES = {'grid', 'list', 'calendar', 'kanban', 'map', 'dashboard', 'chart', - 'timeseries', 'catalog', 'swipe', 'form', 'script'} - -#: W37-T20 (C2) — the cap on `display.script.id`. Script-view ids are minted server-side as -#: `'sv_' + secrets.token_urlsafe(9)` (`routes_script_views.py::create_script_view`), so ~15 chars; -#: 64 is a bound on a hostile payload, not an opinion about the format. -MAX_SCRIPT_ID = 64 - -#: ⭐⭐ WAVE-29 R7 (owner item 10, contracts C3/C4) — THE FORM INTERFACE's stored spec, at -#: `views[].config.display.form`. The public door (`aios-web/api/routes_forms.py`) has read -#: exactly this key since wave 23 and NOTHING HAS EVER BEEN ABLE TO WRITE IT: `form` was not a -#: legal mode and this function had no branch for the key, so every spec a client sent was dropped -#: on the way in. That is D-90 stated precisely — not a broken feature, an unreachable one. -#: -#: ⛔ THE TOKEN IS NOT HERE, AND IT NEVER WILL BE. A share token that rides the wire is a token a -#: browser can CHOOSE, and `routes_forms._resolve` walks tenants and answers with the FIRST match — -#: so one tenant setting its token to another tenant's value would silently receive that tenant's -#: submissions. The token is minted server-side and lives in a bucket no client write can reach; -#: `_clean_form` drops any `token` key that arrives here, rather than validating its shape. -#: ⭐⭐ WAVE-37 T21 (owner items 1/12, rulings R5+R6, contract C3) — THE PER-VIEW CATALOG SPEC, -#: stored at `views[].config.display.catalog`. -#: -#: ⛔⛔ IT IS **`catalog`**, SINGULAR, AND IT IS NOT `catalogs`. Both keys live on `display`, they -#: are one letter apart, and that is the single most likely mistake anybody touching this file will -#: make. They are different things and BOTH are live: -#: · `catalogs` (wave-18 C6) — a LIST of up to 12 print artifacts, each with its own paper, -#: brand and pages of kind cover/intro/section/gallery. Authored CONTENT. UNTOUCHED by T21. -#: · `catalog` (wave-37 C3) — THIS view's catalog SETTINGS: which column is the picture, which -#: is the title (R5: per view, so two catalog views on one database may differ), plus the page -#: order and the free-placed items R6 asks for. -#: Neither replaces the other, and neither validator reads the other's key. -#: -#: ⛔ THE MIGRATION IS A RULE ABOUT WHAT THIS FUNCTION MUST **NOT** DO (R6, C3). An item authored -#: before free placement has no `x`/`y`, and an ABSENT coordinate must read back ABSENT — never -#: normalised to 0. Defaulting to zero would pile every pre-existing item at the origin, which is -#: "a catalog that silently loses its layout is a FAIL, not a migration" stated as code. The -#: RENDERER turns an absent coordinate into flow position N; the host's whole job is to not destroy -#: the distinction it reads. -#: ⚠ `w`/`h` carry the OPPOSITE asymmetry, deliberately: a zero WIDTH is an invisible item, so 0 -#: is refused and absent means "the renderer's default size". A zero X is the top-left corner and is -#: a real, storable position. Two keys of one shape with two rules — the `kanbanClamp` family — -#: which is why each is written out rather than looped. -MAX_CATALOG_SECTIONS = 20 -#: A bound on a hostile payload, not an opinion about the renderer's coordinate space: D owns what -#: the numbers MEAN. Stored to 2 decimals, finer than any print surface resolves. -MAX_CATALOG_COORD = 10000 -#: The page `order` bound. ABSENT => that page's ARRAY INDEX, and `order` WINS where present; ties -#: break by array index. Decided HERE rather than left to the renderer, because two readers guessing -#: differently about page order is exactly the divergence contract C3 exists to close. -MAX_CATALOG_ORDER = 9999 - - -def _clean_catalog_coord(raw, allow_zero): - """One free-placement number, or None. `allow_zero` False is the w/h leg (see the header). - - ⛔ THE ROUNDING IS `floor(v * 100 + 0.5) / 100` AND NOT `round(v, 2)`, AND THAT IS THE WHOLE - POINT OF THIS FUNCTION EXISTING SEPARATELY. Python's `round` is BANKER'S rounding and - JavaScript's `Math.round` is half-up-toward-+Infinity, so the two engines disagree on every - exact half: `round(0.125, 2)` is 0.12 here and 0.13 in the browser. One dragged item landing two - hundredths apart on the two engines is a `display` object that never compares equal — the - silent mirror drift contract C2 exists to prevent, arriving through arithmetic instead of - through a missing key. The expression below is `Math.floor(v * 100 + 0.5) / 100` written in - Python, evaluated in the same IEEE doubles, so the two agree bit for bit. - ⚠ AND AN INTEGRAL RESULT IS EMITTED AS AN `int`: `12.0` serialises as `12.0` in Python and - `12` in JSON.stringify, which is one stored value with two spellings. - """ - if isinstance(raw, bool) or not isinstance(raw, (int, float)): - return None - if raw != raw or raw in (float("inf"), float("-inf")): # NaN / +-inf - return None - val = math.floor(float(raw) * 100 + 0.5) / 100 - if val > MAX_CATALOG_COORD or val < -MAX_CATALOG_COORD: - return None - if not allow_zero and val <= 0: - return None - return int(val) if val == int(val) else val - - -def _clean_catalog_item(raw): - """One free-placed item: `{code, x?, y?, w?, h?}`. `code` REQUIRED; every coordinate optional. - - Key EMISSION ORDER is fixed and mirrored byte-for-byte by `types.ts::cleanCatalogSpec` — an - accepted save has to read back identically on both engines, and this pair is where that is - decided. - """ - if not isinstance(raw, dict): - return None - code = raw.get("code") - if not isinstance(code, str) or not code.strip(): - return None - out = {"code": code.strip()[:CATALOG_CODE_MAX]} - for key, allow_zero in (("x", True), ("y", True), ("w", False), ("h", False)): - val = _clean_catalog_coord(raw.get(key), allow_zero) - if val is not None: - out[key] = val - return out - - -def _clean_catalog_spec(raw, valid_keys): - """C3 — `display.catalog`, fail-closed. See the header block above for what it is NOT. - - Field refs follow the `dateField`/`stackField` rule this module already runs: a ref naming a - deleted column is dropped INDIVIDUALLY and the renderer falls back to its per-mode default. It - never costs the user their pages — the `charts` per-entry-drop precedent, applied to the one - payload here that is AUTHORED rather than derived from the rows. - """ - if not isinstance(raw, dict): - return None - out = {} - for ref in ("imageField", "titleField"): - if raw.get(ref) in valid_keys: - out[ref] = raw[ref] - pages_raw = raw.get("pages") - pages = [] - if isinstance(pages_raw, list): - # ONE cumulative item budget across the whole spec, spent in page order then section order - # — the `_clean_catalogs` construction, so a 40-page catalog cannot smuggle 20,000 items - # past a per-page cap. - budget = MAX_CATALOG_CODES - seen_pages = set() - for raw_page in pages_raw: - if len(pages) >= MAX_CATALOG_PAGES: - break - if not isinstance(raw_page, dict): - continue - pid = str(raw_page.get("id") or "").strip()[:CATALOG_ID_MAX] - if not pid or pid in seen_pages: - continue - seen_pages.add(pid) - page = {"id": pid} - order = raw_page.get("order") - if (isinstance(order, int) and not isinstance(order, bool) - and 0 <= order <= MAX_CATALOG_ORDER): - page["order"] = order - sections_raw = raw_page.get("sections") - sections = [] - if isinstance(sections_raw, list): - seen_sections = set() - for raw_section in sections_raw: - if len(sections) >= MAX_CATALOG_SECTIONS: - break - if not isinstance(raw_section, dict): - continue - sid = str(raw_section.get("id") or "").strip()[:CATALOG_ID_MAX] - if not sid or sid in seen_sections: - continue - seen_sections.add(sid) - section = {"id": sid} - items_raw = raw_section.get("items") - items = [] - if isinstance(items_raw, list) and budget > 0: - for raw_item in items_raw: - item = _clean_catalog_item(raw_item) - if item is None: - continue - items.append(item) - budget -= 1 - if budget <= 0: - break - # ⚠ NOT de-duplicated by `code`. The same product legitimately appears twice in - # one section at two positions — that is what free placement IS — and wave-18's - # "deduped WITHIN a page" rule was written for a LISTING, where a repeat is a - # mistake. Here it is the feature. - if items: - section["items"] = items - sections.append(section) - # An EMPTY section is KEPT: a person adds a New Section and then drags into it, and - # dropping it would delete the thing they just made, between two autosaves. - if sections: - page["sections"] = sections - pages.append(page) - if pages: - out["pages"] = pages - # A spec that reduces to nothing is DROPPED, never stored as `{}`. "No image field, no title - # field, no pages" IS the unconfigured state, and `{}` would be its second spelling — the - # no-churn law every literal-only key in `_clean_display` follows. - return out or None - - -FORM_ACCESS = ('public', 'emails') -MAX_FORM_FIELDS = 60 -MAX_FORM_EMAILS = 200 -MAX_FORM_TITLE, MAX_FORM_DESC, MAX_FORM_SUBMIT = 120, 1000, 60 -MAX_FORM_EMAIL = 254 - -#: The field types a form may COLLECT, as an ALLOW-LIST rather than a list of exclusions — the -#: fail-closed direction, because the cost of the two mistakes is not symmetric. A type missing -#: from here is a question the builder cannot ask yet; a type wrongly present is a public door -#: writing values the column cannot mean (an `image` with no upload channel, a `json` document -#: typed into a text box, a `link` naming a record id a stranger guessed). -#: ⚠ NOT sufficient on its own, and the reason is a shape this codebase has been bitten by before: -#: a METRIC bag rides ANY field type (`core/user_tables.py` — `metric` is not a field kind), so an -#: `int` column can be machine-computed while passing this list. `routes_forms` therefore asks -#: `user_tables.is_computed_cell` as well — one evaluator for "is this computed", reused rather -#: than re-derived ([[one-evaluator-per-question]]). -#: Client mirror: `customer-grid/FormInterface.tsx` FORM_FIELD_TYPES; `verify_forms.py` compares -#: the two files name-for-name. -FORM_FIELD_TYPES = ('text', 'select', 'multiselect', 'int', 'currency', 'pct', 'date', - 'checkbox', 'phone', 'email', 'url', 'rating') - -#: Deliberately looser than a full RFC parse and stricter than `routes_forms._clean_values`' "@ in -#: it": this list decides who MAY SUBMIT, so a typo that silently locks a colleague out is the -#: expensive failure, not an odd address that gets in. -_FORM_EMAIL = re.compile(r'^[^@\s,;]+@[^@\s,;]+\.[^@\s,;]+$') - - -def _clean_form(raw, valid_keys): - """One form spec, fail-closed. Returns None when nothing is configured. - - ⚠ FIELD ORDER IS THE FORM'S OWN and is preserved here, not re-derived from the schema: the - builder let somebody arrange these questions, and sorting them by column order would silently - rearrange a live form every time a column was added (`_public_form` states the same rule from - the serving end). - - ⚠ PARTIAL-DROP, not whole-key drop, and the asymmetry against `swipe` above is deliberate. A - swipe binding is one three-part machine: two of its parts is not a degraded deck, it is a deck - that can never write. A form is a LIST of questions — losing the column behind question three - costs the asker question three, and taking the whole form away because one field was deleted - would be a far larger loss than the one that happened. - """ - if not isinstance(raw, dict): - return None - fields, seen = [], set() - for k in (raw.get('fields') or [])[:MAX_FORM_FIELDS]: - if k in valid_keys and k not in seen: - seen.add(k) - fields.append(k) - out = {} - if fields: - out['fields'] = fields - # A required flag on a question the form no longer asks is not a rule, it is a trap: the - # submitter can never satisfy it and the sentence names a field they cannot see. - req = [k for k in dict.fromkeys(raw.get('required') or []) if k in seen] - if req: - out['required'] = req - for key, cap in (('title', MAX_FORM_TITLE), ('desc', MAX_FORM_DESC), - ('submitLabel', MAX_FORM_SUBMIT)): - text = str(raw.get(key) or '').strip()[:cap] - if text: - out[key] = text - # `public` is the ABSENT default (the `kanbanClamp` law: one way to say one thing), so only - # the restrictive value is ever stored. ⇒ a spec that loses its `access` key fails OPEN, which - # is why `emails` is what gets written rather than a `public: false`. - if raw.get('access') == 'emails': - out['access'] = 'emails' - emails = [] - for e in (raw.get('emails') or [])[:MAX_FORM_EMAILS]: - e = str(e or '').strip().lower()[:MAX_FORM_EMAIL] - if _FORM_EMAIL.match(e) and e not in emails: - emails.append(e) - # Kept even while `access` is public: a person toggling the door open to test it and back - # again must not lose the list of people they typed. It is never served publicly. - if emails: - out['emails'] = emails - return out or None - - -#: C3 — the swipe binding's option cap. `leftOption`/`rightOption` are stored VALUES of a -#: single-select, and `_clean_options` trims every choice to 120 chars, so this is that same -#: number rather than a second opinion about it: a longer string cannot name a real option, and -#: a SHORTER cap here would silently refuse a binding to a legal one. -MAX_SWIPE_OPTION = 120 - -#: C-DISP (wave 2026-08-02): the time-series view's bucket vocabulary and caps, plus the -#: calendar-summary metric cap. types.ts mirrors these as TS_BUCKETS / TS_MAX_LAST_N / -#: TS_MAX_FIELDS / MAX_CALENDAR_METRICS, and cleanDisplay applies the same per-entry drops, -#: so an accepted save reads back byte-identically on both engines. -TS_BUCKETS = {'week', 'month', 'quarter', 'year'} -TS_MAX_LAST_N = 120 -TS_MAX_FIELDS = 12 -MAX_CALENDAR_METRICS = 4 -_ISO_DAY = re.compile(r'^\d{4}-\d{2}-\d{2}$') - -#: C6-CATALOG (wave 18) — the catalog view's vocabulary and caps. types.ts mirrors every name -#: below, and `cleanDisplay` applies the same drops in the SAME ORDER, so an accepted save reads -#: back identically on both engines. The code budget is the order-sensitive one — see -#: `_clean_catalogs`. -MAX_CATALOGS = 12 -MAX_CATALOG_PAGES = 40 -MAX_CATALOG_CODES = 500 # cumulative across ONE catalog's pages, spent in PAGE ORDER -CATALOG_PAPERS = {'letter', 'a4', 'tabloid'} -CATALOG_ORIENTATIONS = {'portrait', 'landscape'} -CATALOG_QUALITIES = {'web', 'print'} -CATALOG_PAGE_KINDS = {'cover', 'intro', 'section', 'gallery'} -CATALOG_COLS = (2, 3, 4) -CATALOG_ID_MAX, CATALOG_NAME_MAX = 40, 80 -CATALOG_TITLE_MAX, CATALOG_BODY_MAX, CATALOG_CODE_MAX = 120, 2000, 60 -_HEX6 = re.compile(r'^#[0-9A-Fa-f]{6}$') - -#: Wave 14 C-ACC ([[loopable-wave14-split]]; rulings R2/R3). Mirrored by types.ts -#: TS_DELTA_KINDS / TS_MAX_CUSTOM_ROWS / TS_MAX_STYLES — the C-DISP byte-identical law. -TS_DELTA_KINDS = ('abs', 'pct', 'yoy', 'ytd') -TS_MAX_CUSTOM_ROWS = 12 -TS_MAX_STYLES = 200 -#: R2 — a formula row's `expr` is stored VERBATIM and NEVER parsed here (evaluation is client -#: law; the client refuses unknown refs/cycles/div-zero itself). The charset wall is the whole -#: server-side contract: row refs `[...]`, arithmetic, numbers — no markup, no control chars. -_TS_EXPR_OK = re.compile(r'^[A-Za-z0-9_ .+\-*/()\[\]]+$') - -#: Old wire value -> the value we store today. Applied AFTER the membership test so an unknown -#: mode is still rejected rather than accidentally aliased. -_LEGACY_MODES = {'dashboard': 'chart'} - -#: Chart kinds a chart-mode view may hold (wave-8 I19, contract C2). Mirrors the client's -#: union. Deliberately small: the owner asked to "start with simple charts" and expand, and a -#: kind the client cannot draw is worse than one that does not exist yet. -#: Wave-16 C-CHARTCAP: + 'table' — the group-by aggregate table (by-rep / by-BU / -#: top-customers, the third Sales block shape). Client renderer: DashboardView's -#: GroupTableView over salesParity.tableFromSpec. -CHART_KINDS = {'bar', 'line', 'area', 'donut', 'kpi', 'table'} -CHART_AGGS = {'sum', 'avg', 'count', 'min', 'max'} - -#: ⭐ WAVE-29 C7 (item 17) — THE COLUMN-SUMMARY vocabulary: what a FIELD's `agg` may be, which is -#: what the grid's totals row and its per-group subtotals compute. ORDERED, because the order is -#: the picker's order; membership tests read it as a tuple perfectly well. -#: -#: ⛔ IT IS NOT `CHART_AGGS` AND THE TWO MUST NOT BE MERGED, however alike they look. `CHART_AGGS` -#: gatekeeps a STORED value with live data behind it (`charts[].agg`, `calendarMetrics[].agg`): -#: `_clean_chart` falls back to 'sum' on an unknown agg and `_clean_display` DROPS a whole -#: calendarMetrics entry, so renaming its 'avg' would silently turn every saved chart into a sum -#: and delete calendar cards, with nothing red. A chart's aggregation and a column's summary are -#: also different questions — one reduces a SERIES, the other a COLUMN — and one list serving both -#: would have to be the intersection of what each can express. -#: -#: ⭐ `average`, NOT `avg`, and the tie is broken by the vocabulary we cannot rename: `ROLLUP_FNS` -#: (`core/user_tables.py`, 16 names, 47 rollups live in production) already spells it `average`, -#: and it is the aggregate vocabulary a user actually reads today. Spelling it `avg` here would -#: give the product two words for one operation on two menus a click apart. -#: -#: ⚠ `median` is net-new: it is in NEITHER `CHART_AGGS` nor `ROLLUP_FNS`, so a Median column -#: summary has no rollup equivalent and this list is NOT a subset of either of its neighbours. -#: -#: ⚠ `count` counts ROWS in the scope (the group, or every matched row) — not non-blank cells. -#: `ROLLUP_FNS` splits that hair three ways (count / counta / countall); a column summary does not, -#: and must not grow a second spelling of it. -#: -#: Client mirror: `customer-grid/iconShapes.ts` FIELD_AGGS — ONE client list, imported by -#: `aggregations.ts` and the field editor rather than re-declared, so the only boundary left to -#: police is this one. `verify_icons.py::agg_parity` reads BOTH FILES and compares them. -FIELD_AGGS = ('sum', 'average', 'median', 'min', 'max', 'count') -MAX_CHARTS = 12 #: per view — a dashboard, not an unbounded render loop -MAX_CHART_TITLE = 60 - -#: Wave-9 I11 (contract C2) — chart customisation, host-validated. -#: -#: `palette` names a colour JOB, never a colour. A browser must not be able to post a raw hex: -#: the four names below map to the four jobs a palette can do (identity / magnitude / polarity) -#: and resolve to brand ramps client-side, so a tenant restyle cannot be defeated by a stored -#: literal. STATUS colours (good/warning/serious/critical) are deliberately NOT selectable — -#: they are reserved signal, and reusing them as "series 4" is how a chart starts lying. -CHART_PALETTES = {'brand', 'categorical', 'sequential', 'diverging'} -CHART_FORMATS = {'auto', 'number', 'currency', 'percent', 'compact'} -MAX_AXIS_LABEL = 40 -#: `size` is the I10 drag. Width is in GRID COLUMNS (a 12-column board), height in px. -CHART_W_RANGE = (1, 12) -CHART_H_RANGE = (120, 800) - - -def _clean_chart(raw, valid_keys): - """One dashboard chart, fail-closed. Returns None if the chart cannot be drawn. - - A chart's `y` is optional (absent = count of rows, which is what "how many customers per - state" means). `x` is NOT: a chart with no category axis has nothing to plot against, and - silently keeping it would put an empty card on the dashboard with no way to tell why. - """ - if not isinstance(raw, dict): - return None - kind = raw.get('kind') - if kind not in CHART_KINDS: - return None - x = raw.get('x') - if x not in valid_keys: - return None # dead category ref -> the chart goes, not the board - cid = raw.get('id') - if not isinstance(cid, str) or not cid.strip(): - return None # the client owns chart ids; an unidentified card - # cannot be edited or removed, so it must not persist - out = {'id': cid.strip()[:64], 'kind': kind, 'x': x, - 'agg': raw.get('agg') if raw.get('agg') in CHART_AGGS else 'sum'} - if raw.get('y') in valid_keys: - out['y'] = raw['y'] - else: - # no measurable column -> the only honest aggregation left is "how many rows" - out['agg'] = 'count' - title = raw.get('title') - if isinstance(title, str) and title.strip(): - out['title'] = title.strip()[:MAX_CHART_TITLE] - - # ── wave-9 I11 (contract C2): customisation ──────────────────────────────────────────── - # `splitBy` is the field whose values become the SERIES. It is deliberately not called - # `colorBy`: that name already means two other things here (`config.colorBy` = row - # colouring, `display.colorField` = map pin colour) and a third sense would be unreadable. - if raw.get('splitBy') in valid_keys and raw['splitBy'] != x: - out['splitBy'] = raw['splitBy'] - # Stacking is only a question once there are series to stack, and only for the two kinds - # that can express it. Anywhere else it is dropped rather than stored as a lie the client - # would have to re-decide. - if out.get('splitBy') and kind in ('bar', 'area') and raw.get('stacked') is True: - out['stacked'] = True - if raw.get('palette') in CHART_PALETTES: - out['palette'] = raw['palette'] - - axis = raw.get('axis') - if isinstance(axis, dict): - # ⛔ ONE y-scale, always. There is no second-axis key here and there must never be: - # two y-scales on one frame can manufacture any correlation you like by rescaling, and - # the honest alternatives are two charts, small multiples, or indexing to a common base. - # Ruled explicitly in contract C2 against the "Tableau versatility" brief. - clean_axis = {} - for side in ('x', 'y'): - spec = axis.get(side) - if not isinstance(spec, dict): - continue - one = {} - lab = spec.get('label') - if isinstance(lab, str) and lab.strip(): - one['label'] = lab.strip()[:MAX_AXIS_LABEL] - if spec.get('format') in CHART_FORMATS: - one['format'] = spec['format'] - if one: - clean_axis[side] = one - if clean_axis: - out['axis'] = clean_axis - - size = raw.get('size') - if isinstance(size, dict): - one = {} - for key, (lo, hi) in (('w', CHART_W_RANGE), ('h', CHART_H_RANGE)): - try: - one[key] = max(lo, min(hi, int(size[key]))) - except (KeyError, TypeError, ValueError): - pass # a partial size is fine: the client defaults the missing axis - if one: - out['size'] = one - - # ── Wave 14 R3 ([[loopable-wave14-split]]): a METRIC chart may carry a PERIOD — the - # trend-over-buckets encoding. Kept only when the chart's value field is measure-backed: - # a category column has no time dimension, and a stored period on it would promise a - # trend the TS channel must refuse. `span` is meaningful only beside `bucket`. - if isinstance(out.get('y'), str) and out['y'].startswith('measure_'): - if raw.get('bucket') in TS_BUCKETS: - out['bucket'] = raw['bucket'] - sp = raw.get('span') - if isinstance(sp, dict): - n = sp.get('lastN') - if (isinstance(n, int) and not isinstance(n, bool) - and 1 <= n <= TS_MAX_LAST_N): - out['span'] = {'lastN': n} - # ── Wave-16 C-CHARTCAP: the YoY companion. Kept ONLY where it can mean something — - # beside a kept bucket (the compare series) or on a sum-of-metric KPI (the delta - # line). Anything else is a stored claim the renderer would have to re-refuse. - # Mirrors the client's cleanCharts rule key for key. - if raw.get('compare') == 'prior_year' and ( - out.get('bucket') or (kind == 'kpi' and out.get('agg') == 'sum')): - out['compare'] = 'prior_year' - return out - - -def _clean_catalog_page(raw, budget): - """C6-CATALOG — one page of a catalog. Returns `(page | None, codes_spent)`. - - `budget` is what is LEFT of the catalog's 500-code allowance. Codes are deduped WITHIN a - page and not across the catalog: a product legitimately appears on a gallery page and again - in its section listing, and de-duplicating globally would silently delete the second - appearance. The budget is spent in page order, so a catalog that runs out loses the TAIL of - its last pages — never a random scatter, and never a page (a page with no products is a - heading the user can still see and fix). - """ - if not isinstance(raw, dict): - return None, 0 - page_id = str(raw.get('id') or '')[:CATALOG_ID_MAX] - kind = raw.get('kind') - if not page_id or kind not in CATALOG_PAGE_KINDS: - return None, 0 - out = {'id': page_id, 'kind': kind} - for key, cap in (('title', CATALOG_TITLE_MAX), ('body', CATALOG_BODY_MAX), - ('imageCode', CATALOG_CODE_MAX)): - v = raw.get(key) - if isinstance(v, str) and v: - out[key] = v[:cap] - products = raw.get('products') - if isinstance(products, list) and budget > 0: - clean_p, seen_p = [], set() - for c in products: - if not isinstance(c, str) or not c: - continue - c = c[:CATALOG_CODE_MAX] - if c in seen_p: - continue - seen_p.add(c) - clean_p.append(c) - if len(clean_p) >= budget: - break - if clean_p: - out['products'] = clean_p - layout = raw.get('layout') - if isinstance(layout, dict): - clean_l = {} - cols = layout.get('cols') - if isinstance(cols, int) and not isinstance(cols, bool) and cols in CATALOG_COLS: - clean_l['cols'] = cols - # The kanbanClamp/tsSparkline asymmetry, one per direction: pack and colour SHOW by - # default (the 2027 catalogue shows both), price does NOT (it shows no prices at all). - # So only the opt-OUT is storable for the first two and only the opt-IN for the third — - # a second spelling of a default is how a round trip starts churning. - if layout.get('showPack') is False: - clean_l['showPack'] = False - if layout.get('showColor') is False: - clean_l['showColor'] = False - if layout.get('showPrice') is True: - clean_l['showPrice'] = True - if clean_l: - out['layout'] = clean_l - return out, len(out.get('products') or ()) - - -def _clean_catalogs(raw, valid_keys): - """C6-CATALOG (wave 18) — `display.catalogs`, fail-closed. Returns a list or None. - - A catalog is a PRINT artifact, so the two structural keys that decide how it paginates - (`paper`, `orientation`) are NORMALISED WITH A DEFAULT rather than dropped: a page box with - no size is not a smaller catalog, it is an unrenderable one. Everything else follows the - house rules — unknown keys dropped, per-entry drops never cost the neighbours, empty - sub-objects omitted entirely (`brand`, `fields`, `layout`, `products`) so an absent key and - an empty one are not two spellings of the same nothing. - - `fields` binds the listing lines to real columns (the 2027 listing prints description / SKU / - pack / colour, and `product_data` carries no pack or colour of its own — the user binds - custom fields). Refs are checked against `valid_keys` HERE and not on the client, the same - split `dateField`/`stackField` already run. - """ - if not isinstance(raw, list): - return None - out = [] - for c in raw: - if len(out) >= MAX_CATALOGS: - break - if not isinstance(c, dict): - continue - cat_id = str(c.get('id') or '')[:CATALOG_ID_MAX] - name = c.get('name') - # An EMPTY name is legal (the user cleared the box and will type again) — an ABSENT one - # is a malformed record. The `tsRows` label rule, same reasoning. - if not cat_id or not isinstance(name, str): - continue - cat = {'id': cat_id, 'name': name[:CATALOG_NAME_MAX]} - cat['paper'] = c['paper'] if c.get('paper') in CATALOG_PAPERS else 'letter' - cat['orientation'] = (c['orientation'] - if c.get('orientation') in CATALOG_ORIENTATIONS else 'portrait') - if c.get('quality') in CATALOG_QUALITIES: - cat['quality'] = c['quality'] - brand = c.get('brand') - if isinstance(brand, dict): - clean_b = {} - for k in ('primary', 'accent'): - v = brand.get(k) - if isinstance(v, str) and _HEX6.match(v): - clean_b[k] = v - company = brand.get('company') - if isinstance(company, str) and company: - clean_b['company'] = company[:CATALOG_NAME_MAX] - # An asset CODE (resolved through C2-ASSET), never a URL: an arbitrary host inside - # print CSS is exactly the tokens-not-values rule this contract carries. - logo = brand.get('logo') - if isinstance(logo, str) and logo: - clean_b['logo'] = logo[:CATALOG_CODE_MAX] - if clean_b: - cat['brand'] = clean_b - binds = c.get('fields') - if isinstance(binds, dict): - clean_bind = {k: binds[k] for k in ('name', 'pack', 'color', 'price') - if binds.get(k) in valid_keys} - if clean_bind: - cat['fields'] = clean_bind - pages, budget = [], MAX_CATALOG_CODES - raw_pages = c.get('pages') - if isinstance(raw_pages, list): - for p in raw_pages: - if len(pages) >= MAX_CATALOG_PAGES: - break - page, spent = _clean_catalog_page(p, budget) - if page is None: - continue - budget -= spent - pages.append(page) - # ALWAYS emitted, even empty: a catalog with no pages yet is the state every catalog - # starts in, and dropping the key would make "new" and "corrupt" the same wire value. - cat['pages'] = pages - out.append(cat) - return out or None - - -def _clean_display(raw, valid_keys): - """Structural passthrough for a view's `config.display` (wave-6 item 10), fail-closed. - - `{mode, dateField?, stackField?, titleField?, colorField?, sizeField?, charts?}` — mode - must be a known non-grid mode (grid is the absent default, so storing it would be a second - way to say nothing); every field ref must name a field this table has (a ref to a deleted - field is DROPPED and the client falls back to its per-mode default); unknown keys are - dropped. What each mode MEANS — calendar wants a date-family field, kanban a select-family - stack, map a single-select to colour by and a numeric to size by — is the client's - business: it is the only layer that renders them, and a wrong-typed ref degrades to that - surface's default rather than to an error (the `_clean_window` split). - - Wave-8 (contract C2) adds the map encodings (`colorField` I3, `sizeField` I5) and - dashboard `charts` (I19). A chart whose x/y names a deleted field is dropped INDIVIDUALLY — - never the whole array, because losing one column should not cost the user a dashboard they - spent time building. - - W33-T45 (contract C1, ruling R5) adds `published` + `publishAccess`. ⚠ THE HALF OF THE ROUND - TRIP THIS FUNCTION CANNOT ENFORCE: `customer-grid/types.ts::cleanDisplay` is a SECOND - normalizer, in the browser, which rebuilds the config key by key on every autosave and drops - anything it does not name. A key accepted here and unknown there dies on the next column - resize — silently, because the client then POSTs the stripped config and the host REPLACES - the stored one. "The host accepts it" is half a round trip; that file is the other half. - """ - if not isinstance(raw, dict): - return None - mode = raw.get('mode') - if mode not in DISPLAY_MODES or mode == 'grid': - return None - # Wave-9 I10 (C2): normalise the legacy wire value AFTER the membership test, so an unknown - # mode is still rejected rather than accidentally aliased into a real one. Every already - # saved 'dashboard' view reads back as 'chart' from here on; nothing writes 'dashboard'. - mode = _LEGACY_MODES.get(mode, mode) - out = {'mode': mode} - # ⭐ W37-T28 (owner item 15) — `coordField`: ONE column holding `"lat,lon"`, so a map is no - # longer a customers-only feature. It joins this loop rather than getting a clause of its own - # because it IS a field ref with the same drop-if-deleted rule; `customer-grid/types.ts` - # gains it in the SAME change, which is contract C2's whole point. - for ref in ('dateField', 'stackField', 'titleField', 'colorField', 'sizeField', 'coordField'): - if raw.get(ref) in valid_keys: - out[ref] = raw[ref] - # ── C-DISP (wave 2026-08-02) ───────────────────────────────────────────────────────── - # kanbanClamp: stored ONLY as the literal opt-OUT. Absent means clamped — the new - # standardized default — so storing True would be a second way to say nothing (the same - # rule that keeps mode:'grid' out of the store). - if raw.get('kanbanClamp') is False: - out['kanbanClamp'] = False - if raw.get('calendarMode') in ('records', 'summary'): - out['calendarMode'] = raw['calendarMode'] - metrics = raw.get('calendarMetrics') - if isinstance(metrics, list): - clean_m, seen_m = [], set() - for m in metrics[:MAX_CALENDAR_METRICS]: - # Dropped INDIVIDUALLY (the charts precedent): one dead metric must not cost the - # user the summary card they configured around it. - if not isinstance(m, dict): - continue - mid = str(m.get('id') or '')[:40] - if (not mid or mid in seen_m or m.get('field') not in valid_keys - or m.get('agg') not in CHART_AGGS): - continue - seen_m.add(mid) - clean_m.append({'id': mid, 'field': m['field'], 'agg': m['agg']}) - if clean_m: - out['calendarMetrics'] = clean_m - if raw.get('tsBucket') in TS_BUCKETS: - out['tsBucket'] = raw['tsBucket'] - span = raw.get('tsSpan') - if isinstance(span, dict): - clean_span = {} - n = span.get('lastN') - if isinstance(n, int) and not isinstance(n, bool) and 1 <= n <= TS_MAX_LAST_N: - clean_span['lastN'] = n - else: - f, t = span.get('from'), span.get('to') - f = f if isinstance(f, str) and _ISO_DAY.match(f) else None - t = t if isinstance(t, str) and _ISO_DAY.match(t) else None - if f and t and f > t: - f, t = t, f - if f: - clean_span['from'] = f - if t: - clean_span['to'] = t - if clean_span: - out['tsSpan'] = clean_span - ts_fields = raw.get('tsFields') - if isinstance(ts_fields, list): - clean_f, seen_f = [], set() - for k in ts_fields[:TS_MAX_FIELDS]: - if k in valid_keys and k not in seen_f: - seen_f.add(k) - clean_f.append(k) - if clean_f: - out['tsFields'] = clean_f - # ── Wave 14 C-ACC ([[loopable-wave14-split]] R2; items 17/18) ──────────────────────── - deltas = raw.get('tsDeltas') - if isinstance(deltas, list): - clean_d, seen_d = [], set() - for d in deltas: - if d in TS_DELTA_KINDS and d not in seen_d: - seen_d.add(d) - clean_d.append(d) - if clean_d: - out['tsDeltas'] = clean_d - # Gridlines: stored ONLY as the literal opt-OUT (absent = shown), sparkline ONLY as the - # literal opt-IN (absent = off) — the kanbanClamp asymmetry, one per direction. - if raw.get('tsGridlines') is False: - out['tsGridlines'] = False - if raw.get('tsSparkline') is True: - out['tsSparkline'] = True - rows = raw.get('tsRows') - if isinstance(rows, list): - clean_r, seen_r = [], set() - for r in rows[:TS_MAX_CUSTOM_ROWS]: - if not isinstance(r, dict): - continue - rid = str(r.get('id') or '')[:40] - r_kind = r.get('kind') - if not rid or rid in seen_r or r_kind not in ('note', 'formula'): - continue - label = r.get('label') - if not isinstance(label, str): - continue # ABSENT label = malformed; an EMPTY one is a legal spacer row - # (GRID's dated asymmetry amendments, 2026-08-02) - one = {'id': rid, 'kind': r_kind, 'label': label.strip()[:120]} - if r_kind == 'formula': - expr = r.get('expr') - if (isinstance(expr, str) and expr.strip() - and len(expr) <= 200 and _TS_EXPR_OK.match(expr)): - one['expr'] = expr - # else: keep the ROW, drop the EXPR — it renders "—". Vanishing the row - # would delete the user's label to punish their arithmetic (GRID's dated - # asymmetry amendment; the calendarMetrics per-entry-drop precedent). - seen_r.add(rid) - clean_r.append(one) - if clean_r: - out['tsRows'] = clean_r - styles = raw.get('tsStyles') - if isinstance(styles, dict): - clean_s = {} - for s_key, s_val in styles.items(): - if len(clean_s) >= TS_MAX_STYLES: - break # capped, not truncated silently: the gate names this - if not isinstance(s_key, str) or not s_key or len(s_key) > 96: - continue # key = rowId or "rowId:colKey" — the client's grammar - if not isinstance(s_val, dict): - continue - one = {} - if s_val.get('bold') is True: - one['bold'] = True - if s_val.get('line') is True: - one['line'] = True - if one: - clean_s[s_key] = one - if clean_s: - out['tsStyles'] = clean_s - charts = raw.get('charts') - if isinstance(charts, list): - clean = [c for c in (_clean_chart(x, valid_keys) for x in charts[:MAX_CHARTS]) if c] - # de-dupe by id: two cards sharing an id are one card as far as the client's keyed - # render is concerned, and the second would silently shadow the first - seen, uniq = set(), [] - for c in clean: - if c['id'] in seen: - continue - seen.add(c['id']) - uniq.append(c) - if uniq: - out['charts'] = uniq - # ── C6-CATALOG (wave 18) ───────────────────────────────────────────────────────────── - catalogs = _clean_catalogs(raw.get('catalogs'), valid_keys) - if catalogs: - out['catalogs'] = catalogs - # ── ⭐⭐ WAVE-37 C3 (T21) — `catalog`, SINGULAR, and it is NOT the line above it. See the - # header on `_clean_catalog_spec` for the two-keys-one-letter-apart warning; this call site is - # where a reader is most likely to "fix" one into the other. - catalog = _clean_catalog_spec(raw.get('catalog'), valid_keys) - if catalog: - out['catalog'] = catalog - # ── ⭐ WAVE-27 C3 (item 8 / R2): the swipe binding ──────────────────────────────────── - # `{fieldKey, leftOption, rightOption}` — WHOLE-KEY drop, never a partial one, and that - # asymmetry against `charts`/`calendarMetrics` above is the point rather than an oversight. - # Those are LISTS of independent cards, so losing one entry costs the user one card. This is - # a single three-part BINDING: a swipe view holding a fieldKey with one option, or two - # options and no field, is not a degraded swipe view — it is a deck that can never write - # anything, rendered as though it were configured. Dropping the key entirely puts the view - # back in its honest unconfigured state, which is the one state the client has a UI for. - # - # ⚠ What this CANNOT check, deliberately, and why the client must: whether `fieldKey` names - # a SELECT, and whether the two options are still in that select's vocabulary. `valid_keys` - # is a key set, and the docstring above draws this exact line — "what each mode MEANS ... is - # the client's business". So SwipeView owns three losses this function is blind to (field - # deleted, field retyped away from select, option removed) and must SHOW each one rather - # than fall back to the first option, per the `viewModes.tsx` house rule. - swipe = raw.get('swipe') - if isinstance(swipe, dict): - f_key = swipe.get('fieldKey') - left, right = swipe.get('leftOption'), swipe.get('rightOption') - ok = (f_key in valid_keys - and isinstance(left, str) and isinstance(right, str)) - if ok: - left, right = left.strip()[:MAX_SWIPE_OPTION], right.strip()[:MAX_SWIPE_OPTION] - # Both non-empty, and DISTINCT: one option on both sides is a deck whose two - # gestures do the same thing, which is two spellings of one state (the - # `kanbanClamp` law) wearing a control that promises a choice. - if left and right and left.casefold() != right.casefold(): - out['swipe'] = {'fieldKey': f_key, 'leftOption': left, 'rightOption': right} - # ── ⭐⭐ WAVE-37 T20 (owner item 10 / R2, contract C2): WHICH SCRIPT THIS VIEW RENDERS ── - # - # `{id}` — the row in E's per-database script store (`/api/v1/script-views`) whose source this - # view draws. WHOLE-KEY drop, the `swipe` construction rather than the `charts` one: a script - # view with no id is not a degraded script view, it is a panel with nothing to run, and the - # honest state for that is the unconfigured one the client already has a surface for. - # - # ⛔ THE ID IS DELIBERATELY NOT CHECKED AGAINST `valid_keys`, AND THAT IS NOT AN OVERSIGHT. - # `valid_keys` is this table's FIELD keys; a script id names a record in a DIFFERENT store that - # this function cannot reach. So the host bounds the shape and nothing else, and the renderer - # owns the one loss this is blind to — the script row was deleted — which it must SHOW rather - # than fall back to another script (the `viewModes.tsx` house rule `_clean_swipe`'s note cites). - # ⚠ Mirrored key-for-key by `customer-grid/types.ts::cleanDisplay`. That file rebuilds the - # config key by key on every autosave, so a key accepted HERE and unknown THERE is erased on the - # next column resize — which is the half of the round trip this docstring's W33 note warns about. - script = raw.get('script') - if isinstance(script, dict): - s_id = script.get('id') - if isinstance(s_id, str) and s_id.strip(): - out['script'] = {'id': s_id.strip()[:MAX_SCRIPT_ID]} - # ── ⭐⭐ WAVE-29 R7 (item 10): the FORM spec — see `_clean_form` for why the token is not here. - form = _clean_form(raw.get('form'), valid_keys) - if form: - out['form'] = form - # ── ⭐⭐ W33-T45 / CONTRACT C1 / RULING R5 (owner item 8b): IS THIS INTERFACE PUBLISHED ──── - # - # ⛔ TWO KEYS LIVE HERE AND TWO DELIBERATELY DO NOT. The `published` flag and the sharer's - # `public | password` choice are DISPLAY state — the view says what it is, the client renders - # a badge from it, and it travels with the view like every other key in this dict. The SECRET - # TOKEN and the PASSPHRASE HASH do not: they live in the server-only bucket, exactly as - # `routes_forms.py`'s `TOKENS_KEY` holds the form token. `config.display` is echoed back to - # every user who can open the view, so a token in here is a token published to the audience - # the password was meant to exclude. `_clean_form` above carries the same rule and the same - # reason; this is the second door, not a new one. - # - # ⛔ AND THE COERCION IS FAIL-CLOSED, WHICH IS WHY THIS IS NOT A BARE ALLOWLIST. A plain - # allowlist drops an unrecognised `publishAccess` and KEEPS `published: True` — leaving a - # published view with no stated access, i.e. a third state neither the ruling nor the client - # has a meaning for, on the one key where guessing wrong publishes a tenant's data to the - # open internet. So: a published view ALWAYS carries an access, and anything that is not the - # literal `'public'` reads as `'password'`. The unpublished case stores nothing at all — - # absent means unpublished, and a `published: False` would be the second way to say nothing - # that `kanbanClamp` and `mode: 'grid'` are both here to forbid. - if raw.get('published') is True: - out['published'] = True - out['publishAccess'] = 'public' if raw.get('publishAccess') == 'public' else 'password' - return out - - -#: FOLDERS over the saved views / cohorts sidebars (wave-8 I11, contract C4). -#: -#: ⚠ Folder membership is stored as a SIDE MAP (`itemFolders`), not as a `folderId` ON each -#: view or cohort — a deliberate amendment to C4's first wording, recorded in the split doc. -#: Two reasons. (1) A cohort lives in a DIFFERENT store (`customer_cohorts`, keyed by cohort -#: id) and adding a `folders` key beside those ids would collide with a cohort whose generated -#: id happened to be 'folders'. (2) Folder placement is a per-user ORGANISING act, not part of -#: what a view IS: keeping it out of the view config means duplicating or exporting a view does -#: not drag a folder reference along with it. One map, one home, both surfaces. -FOLDER_SURFACES = {"views", "cohorts"} -MAX_FOLDERS = 60 -MAX_FOLDER_NAME = 80 - -#: Wave-9 I15 (contract C5) — a user-chosen folder icon, as {shape, tone}. -#: -#: Both halves are WHITELISTS, never free values: `shape` names geometry the client already -#: draws (one source, `iconShapes.ts`, read by both painters) and `tone` names a palette token, -#: not a colour — so a browser cannot post a hex and defeat a tenant restyle, and a shape the -#: client cannot render can never reach the store. -#: ⚠ MIRRORS CLIENT'S `iconShapes.ts` ENUMERATION EXACTLY (C5: CLIENT enumerates, HOST mirrors — -#: posted in the split doc 2026-07-29, HOST adopted it the same day, replacing a provisional -#: 12-shape guess of mine that contained shapes the client cannot draw). Do not extend this set -#: without the matching client geometry: an unknown shape falls back to the default folder mark, -#: which is also I14's "existing folders get the folder icon" for every pre-wave-9 folder. -FOLDER_ICON_SHAPES = {"folder", "star", "flag", "tag", "bookmark", "box", "circle", "square"} -#: Tones are the C1 pastels — FILLS ONLY, never text (the standing palette rule). 'grey' is the -#: default, and is CLIENT's key name: not 'neutral', which is what HOST first guessed. -FOLDER_ICON_TONES = {"blue", "green", "yellow", "red", "grey"} -FOLDER_ICON_DEFAULT_TONE = "grey" - - -#: Wave-9 I17 (contract C4) — who may EDIT a saved view. -#: -#: ⚠ READ THIS BEFORE BUILDING ON IT. Views are stored PER USER today -#: (`core/table_store.TableStore.workspace` reads `store.get(table_key)[username]`), so one -#: user's views are invisible to every other user and "collaborative" has nothing to act on -#: yet. This validator is therefore CORRECT-BUT-INERT plumbing: it makes the setting durable -#: and fail-closed now, so that when shared views land the permission does not need a data -#: migration and no saved view is retro-restricted. It does NOT make anything shared, and -#: nothing in the app currently reads it to grant or deny cross-user access. -#: Recorded as the C4 amendment in .claude/wiki/research/grid-wave9-split.md. -VIEW_EDIT_MODES = {'personal', 'collaborative', 'users'} -MAX_VIEW_USERS = 50 - - -def clean_view_permissions(raw, default, known_users=None): - """{edit, users?} — fail-closed on both halves. - - `default` is supplied by the CALLER because it splits by path, and that split is a - permission rule rather than a formatting one: absent on a view that already exists means a - pre-wave-9 view and must stay 'collaborative' (retro-restricting somebody's saved view is a - silent takeaway), while absent on CREATE must be 'personal' (a new view must never be - anyone-can-edit purely by omission). - - `known_users` (when given) is the real account list: an unknown name is DROPPED, and an - 'users' grant left with nobody in it collapses to 'personal' rather than to everyone. - """ - mode = (raw or {}).get('edit') if isinstance(raw, dict) else None - if mode not in VIEW_EDIT_MODES: - mode = default if default in VIEW_EDIT_MODES else 'personal' - if mode != 'users': - return {'edit': mode} - names, seen = [], set() - for u in list((raw or {}).get('users') or [])[:MAX_VIEW_USERS]: - u = str(u or '').strip() - if not u or u.lower() in seen: - continue - if known_users is not None and u not in known_users: - continue # fail-closed: a name we cannot resolve grants nothing - seen.add(u.lower()) - names.append(u) - if not names: - return {'edit': 'personal'} # an empty grant is NOT "everyone" - return {'edit': 'users', 'users': names} - - -def clean_folder_icon(raw): - """{shape, tone} or None. Fail-closed on both halves, independently. - - A folder with a valid shape but a junk tone keeps the shape and defaults the tone rather - than losing the icon entirely — losing a user's pick because one half was wrong is the kind - of silent data loss the folder events already avoid elsewhere. - """ - if not isinstance(raw, dict): - return None - shape = raw.get("shape") - if shape not in FOLDER_ICON_SHAPES: - return None - tone = raw.get("tone") - return {"shape": shape, - "tone": tone if tone in FOLDER_ICON_TONES else FOLDER_ICON_DEFAULT_TONE} - - -def clean_folders(raw): - """Validate the per-surface folder lists, fail-closed. {surface: [{id, name, order}]}.""" - out = {} - for surface in FOLDER_SURFACES: - items, seen = [], set() - for f in list((raw or {}).get(surface) or [])[:MAX_FOLDERS]: - if not isinstance(f, dict): - continue - fid = str(f.get("id") or "").strip()[:80] - name = str(f.get("name") or "").strip()[:MAX_FOLDER_NAME] - if not fid or not name or fid in seen: - continue # an unidentified or unnamed folder cannot be shown or edited - seen.add(fid) - try: - order = int(f.get("order", len(items))) - except (TypeError, ValueError): - order = len(items) - row = {"id": fid, "name": name, "order": order} - icon = clean_folder_icon(f.get("icon")) # wave-9 I15 (C5); absent = default mark - if icon: - row["icon"] = icon - items.append(row) - items.sort(key=lambda x: x["order"]) - for i, f in enumerate(items): - f["order"] = i # re-index so `order` is always dense and total - if items: - out[surface] = items - return out - - -#: ⭐⭐ WAVE 32 · OWNER ITEM 20 (`W32-T27`, raised by SESSION C as ASK C-16) — "FILED AT ROOT". -#: -#: ⛔ THE DEFECT IS THAT ROOT WAS REPRESENTED BY *ABSENCE*, AND ABSENCE CANNOT HOLD TWO FACTS. -#: "this arrived by grant and was never filed" and "the receiver deliberately dragged this OUT of -#: the Shared group" were the same stored state — nothing — so the client had to GUESS, and -#: `folders.ts::groupByFolder` guessed "Shared". That is why only folder→folder moves appeared to -#: work: **the root bucket was unreachable for a shared view by construction.** -#: -#: ⚠ A RESERVED FOLDER ID, NOT A NEW FIELD, deliberately. The placement map is `{itemId: folderId}` -#: and every reader on both sides already understands it; a parallel "filedAtRoot" set would be a -#: second source of truth for one question, and the two would disagree the first time one of them -#: was written without the other. This id names no folder BY DESIGN and is therefore exempt from -#: the folder-exists test below — it is the one value that means "no folder, on purpose". -#: ⚠ Spelled `ROOT_FOLDER_ID` on the client (`customer-grid/folders.ts`, C's file). Two spellings -#: of one constant is [[a-constant-two-features-share]]; `verify_folders`/`verify_api` assert they -#: agree rather than a comment asking nicely. -ROOT_PLACEMENT = "__root__" - - -def clean_item_folders(raw, folders, valid_ids): - """{surface: {itemId: folderId}} — dropping any placement whose ITEM or FOLDER is gone. - - This is what makes a deleted folder's contents fall back to the root rather than vanish: - nothing stores "this item is in no folder", so an unresolvable placement simply disappears - and the item renders at the top level. Same for an item that was deleted elsewhere — its - stale placement can never resurrect it, because the sidebars render ITEMS and consult this - map, never the other way round. - - ⭐⭐ WAVE 32 — THE PARAGRAPH ABOVE STATES THE FEATURE AND THE BUG IN ONE SENTENCE, and it took - owner item 20 to notice they were the same mechanism. *"Nothing stores 'this item is in no - folder', so an unresolvable placement simply disappears"* is exactly right for a DELETED FOLDER - (its contents should fall to the root) and exactly wrong for a SHARED VIEW (falling back means - falling back INTO the Shared group, which is where it started). `ROOT_PLACEMENT` is the value - that survives this function so the second case can be said out loud. - """ - out = {} - for surface in FOLDER_SURFACES: - fids = {f["id"] for f in (folders or {}).get(surface, [])} - ok = {} - for item_id, fid in ((raw or {}).get(surface) or {}).items(): - if not isinstance(item_id, str) or not isinstance(fid, str): - continue - # ⛔ `fid == ROOT_PLACEMENT` FIRST, and it is NOT in `fids` — it names no folder, which - # is the whole point. Without this clause the value is written by `item_move` and - # scrubbed here on the way back out, so the mark would be stored and instantly lost: - # the two halves are ONE change and shipping either alone is worse than shipping - # neither ([[lost-write-looks-like-failed-read]]). - if item_id in (valid_ids or {}).get(surface, ()) and (fid == ROOT_PLACEMENT - or fid in fids): - ok[item_id[:120]] = fid - if ok: - out[surface] = ok - return out - - -def view_refs_of(nodes): - """C4 — every view id a tree REFERS to, deduped, in order. Mirrors `types.ts::viewRefsOf`. - - ⚠ An INACTIVE leaf (no view chosen yet) is not a reference, on both engines: counting one - would refuse a save the moment somebody opened the condition and had not picked anything. - """ - out = [] - for node in nodes or []: - if not isinstance(node, dict): - continue - if 'children' in node: - for vid in view_refs_of(node.get('children')): - if vid not in out: - out.append(vid) - continue - if node.get('colId') != VIEW_FIELD: - continue - vid = str(node.get('value') or '').strip()[:MAX_VIEW_ID] - if vid and vid not in out: - out.append(vid) - return out - - -def view_filter_cycle(view_id, nodes, filters_of): - """⭐⭐ W37-T26 / C4 — THE CYCLE REFUSAL's host half. Mirrors `types.ts::viewFilterCycle`. - - View X filtered on "is in View Y" where Y is filtered on X cannot be resolved by anything: - X needs Y needs X. Returns the CYCLE PATH (ids, starting and ending at `view_id`) or None. - - ⛔ THE CLIENT REFUSES THE SAME SHAPE AND THAT IS NOT DUPLICATION. The browser's guard is what - a PERSON sees and is the only one that can explain itself; this one is what makes the rule true - for a caller that POSTs a view directly, which is every guard-in-the-browser's blind spot. - ⚠ A view referring to ITSELF is a cycle of length one and is caught by the same walk — also - the case a person reaches most easily, by duplicating a view. - """ - def walk(current, path, seen): - refs = view_refs_of(nodes if current == view_id else filters_of(current)) - for nxt in refs: - if nxt == view_id: - return path + [nxt] - if nxt in seen: - continue - seen.add(nxt) - found = walk(nxt, path + [nxt], seen) - if found: - return found - return None - - return walk(view_id, [view_id], {view_id}) - - -def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None, - visible_view_ids=None): - """Recursively validate an UNTRUSTED filter tree (conditions + nested groups). - - Returns a clean tree of leaf conditions ({colId, op, value, value2}) and - groups ({conj, children}). Module-agnostic on purpose: any module embedding - the grid validates its own view state through this one function. - - Fail-closed PER NODE: anything unrecognised is DROPPED rather than raised — - the same contract the rest of the view sanitiser follows, so one bad rule can - never cost a user their whole saved view. Depth, per-level width and total - node count are all capped: the tree is re-evaluated for every row on every - render, so an unbounded structure would be a persistent client-side DoS. - Empty groups are dropped (they carry no meaning once persisted). - - `cohort_ids` is the set of cohorts the CALLER may see. A cohort leaf naming anything else is - DROPPED here rather than left for the engine — a deleted cohort would otherwise leave a - condition that can only match nothing, so `List is not [deleted]` would show an empty table - forever with no way to tell why. `None` means this host has no cohorts, and then every - cohort leaf is dropped: fail-closed, like every other unknown key. - - ⭐⭐ W37-T26 (C4) — `visible_view_ids` is the same parameter one leaf over: the views this - caller may name in a `__view__` membership leaf. A leaf naming anything else is DROPPED here - for the identical reason, and `None` (this host has no views to offer) drops every one. - ⚠ DEFAULTS TO `None` SO EVERY EXISTING CALLER IS BYTE-IDENTICAL. Three callers pass nothing - (`routes_admin`, `routes_query` twice) and each of them is a surface with no saved-view - vocabulary of its own; under the fail-closed rule they drop the leaf, which is correct rather - than merely safe. - """ - if budget is None: - budget = [MAX_FILTER_NODES] - out = [] - for node in list(raw or [])[:MAX_FILTER_SIBLINGS]: - if budget[0] <= 0: - break - if not isinstance(node, dict): - continue - if isinstance(node.get('children'), list): # a condition GROUP - if depth >= MAX_FILTER_DEPTH: - continue # too deep -> drop - budget[0] -= 1 - children = clean_filter_tree(node['children'], valid_keys, - depth + 1, budget, cohort_ids, visible_view_ids) - if children: - out.append({'conj': 'or' if node.get('conj') == 'or' else 'and', - 'children': children}) - continue - if node.get('colId') == COHORT_FIELD: # a cohort-membership leaf - op = COHORT_OP_ALIASES.get(node.get('op'), node.get('op')) - named = parse_cohort_ids(node.get('value')) - # ALL of them, or the leaf goes. A set that quietly lost a member asks a DIFFERENT - # question, and for `noneOf` a strictly wider one: `is none of [A, B]` degrading to - # `is none of [A]` would show every row in B under a count nobody would doubt. This - # is the same all-or-nothing the single-cohort leaf already had, extended to a set. - if op in COHORT_OPS and named and all(c in (cohort_ids or ()) for c in named): - budget[0] -= 1 - out.append({'colId': COHORT_FIELD, 'op': op, - 'value': ','.join(named), 'value2': ''}) - continue - if node.get('colId') == VIEW_FIELD: # C4: a view-membership leaf - # ⛔ THE SAME ALL-OR-NOTHING THE COHORT ARM ABOVE APPLIES, AND FOR THE SAME REASON. - # A leaf naming a view this caller cannot see is DROPPED rather than kept, because a - # kept one can only ever match nothing: `View is not [a view you cannot see]` would - # show an empty table forever with no way to tell why. Dropping it puts the view back - # in its honest unfiltered state, which is what `clean_filter_tree`'s own docstring - # already promises for cohorts. - # ⚠ `visible_views` is what the caller passes; `None` means this host does not know, - # and then EVERY view leaf is dropped — fail-closed, like every other unknown key. - v_op = node.get('op') - v_id = str(node.get('value') or '').strip()[:MAX_VIEW_ID] - if v_op in VIEW_OPS and v_id and v_id in (visible_view_ids or ()): - budget[0] -= 1 - out.append({'colId': VIEW_FIELD, 'op': v_op, 'value': v_id, 'value2': ''}) - continue - if node.get('colId') in valid_keys and node.get('op') in FILTER_OPS: - budget[0] -= 1 - # `or ''` would be wrong here: it maps every FALSY value to '', and '' is the - # signal for "inactive". A numeric 0 (or 0.0, or False) is a real value the client - # treats as active — `0 === ""` is false in TS — so `revenue = 0` would silently - # stop filtering and show every row instead of the zero-revenue ones. - val, val2 = node.get('value'), node.get('value2') - leaf = {'colId': node['colId'], 'op': node['op'], - 'value': ('' if val is None else str(val))[:500], - 'value2': ('' if val2 is None else str(val2))[:500]} - # CG-8. A MEASURE condition ("Sales, in the last 90 days, > 5,000") carries two - # extra members: a stable client-generated `id`, which is how the server's answer - # finds its way back to the condition that asked (positional matching silently - # re-associates every answer the moment a user deletes a condition), and the - # `window`. Emitted ONLY when the input has them — a column condition's cleaned - # shape is unchanged, so every persisted view deserialises byte-identically and - # `clean_filter_tree` stays idempotent (verify_filter_engine.py asserts that by - # exact structural comparison). - rid = node.get('id') - if rid not in (None, ''): - leaf['id'] = str(rid)[:64] - window = _clean_window(node.get('window')) - if window is not None: - leaf['window'] = window - # Owner items 3 + 4, carried under the SAME rule as CG-8's `id`/`window`: emitted - # only when the input has them, so a plain column condition's cleaned shape is - # byte-identical to what it always was and `clean_filter_tree` stays idempotent - # (verify_filter_engine.py asserts that by exact structural comparison). Drop the - # carry-through and the next autosave silently strips a date condition back to a - # bare comparison against an empty value — i.e. back to INACTIVE. - date_window = _clean_window(node.get('dateWindow')) - if date_window is not None: - leaf['dateWindow'] = date_window - mode = node.get('dateMode') - # SHAPE only. An unrecognised mode survives here and is refused by - # `windows.resolve_anchor`, which returns None and makes the condition match - # NOTHING — the same split as `_clean_window`, and the reason this module can stay - # free of the date vocabulary it would otherwise have to keep in step. - if isinstance(mode, str) and 0 < len(mode) <= 40: - leaf['dateMode'] = mode - rhs = _clean_rhs(node.get('rhs'), valid_keys) - if rhs is not None: - leaf['rhs'] = rhs - out.append(leaf) - return out - - -def _default_view_config(fields): - # ⭐⭐ W30-T41's SERVER HALF (F's ask F-1, answered by D — this file is D's fence). - # - # ⛔ THE SECOND ARM USED TO BE `or field["source"] == "overlay"`, AND IT SWALLOWED THE FIRST - # ONE FOR EVERY CONNECTED COLUMN. `user_tables._clean_field` stamps `source: "overlay"` on - # every `ut_` field, so on an Odoo grid the arm was true for ALL of them and `default: False` - # meant nothing: `odoo_id`, `state`, `customer_link` and `partner_id` opened SHOWN however - # they were declared. The exception had become the rule ([[fallback-that-became-the-rule]]), - # and it is the same predicate `useGridColumns.isDefaultVisible` carried on the client. - # - # ⚠ AND THE TWO HALVES MUST MOVE TOGETHER, which is why this is not cosmetic. `CustomerGrid` - # compares the stored view against its own `defaultViewConfig` by JSON equality; with the - # client fixed (T41) and this left alone, the system view would differ from the client's - # default on every render — a view that looks permanently dirty and autosaves forever, which - # is the failure `verify_filter_engine`'s key-ORDER check exists to prevent, one level down. - # - # ⚠ A USER-CREATED COLUMN IS UNAFFECTED, and that is why the fix is a DELETION rather than a - # carve-out for the four Odoo keys: it carries no `default` key at all, so `is not False` - # keeps it visible. On the main Customer grid exactly one field moves — `notes`, which asks - # to be hidden in its own declaration and was being shown against it. - shown = [field["key"] for field in fields if field.get("default") is not False] - hidden = [field["key"] for field in fields if field["key"] not in shown] - return { - # `filters` is the ROOT of the filter tree: leaf conditions and/or nested - # condition groups ({conj, children}); `filterConj` joins the root level. - "filters": [], "filterConj": "and", - "sorts": [], "groupBy": None, "colorBy": None, - "rowHeightMode": "short", "order": shown + hidden, "visible": shown, - "widths": {}, "memberPids": [], - } - - -#: Wave 17 R1 / C-LOCKV — the `kind` a PROJECTED locked view wears. A cohort is not a separate -#: kind of object any more: it is a saved view whose rows are a hand-curated set. -LOCKED_VIEW_KIND = 'locked' - - -def locked_view_projection(entry, base_config): - """One cohort -> the saved-view row that IS it (wave 17 R1, contract C-LOCKV). - - ⛔ THE LOCK IS THE VIEW'S IDENTITY, NOT ITS CONFIGURATION. `config.cohortLock` names the - view's OWN id, which is what makes the shipped engine law (`useVisibleRows`: intersect the - named set FIRST, unconditionally, and match NOTHING when the membership is unresolvable) do - all the work with no second mechanism. Membership itself is NEVER copied in here — it stays - in `customer_cohorts` and travels as `workspace.lists`, because a per-reader-scoped - collection inside a client-writable `config` is deleted by the next autosave (see the - contract's reason 2). - - ⚠ `locked: True` is the LEGACY "undeletable/mode-frozen" flag and is deliberately NOT set: - these views are ordinary in every respect the owner asked for — reorder, folder, sort, - filter, change display mode. The lock mark in the rail is driven by `kind`. - """ - return { - 'id': entry['id'], - 'name': entry.get('name') or entry['id'], - 'kind': LOCKED_VIEW_KIND, - 'config': {**base_config, 'cohortLock': entry['id']}, - } - - -#: ⭐ WAVE-27 item 27 (owner ruling R8) — the IG "Overview" view's CURATED COLUMNS, in the -#: owner's own order: handle, followers, engagement, location, last enriched. -#: -#: Written as candidates rather than as a requirement. The template registry REFUSES a template -#: whose columns the target lacks (`view_templates.missing_columns`) because applying one writes -#: the user's own views and a filter on a missing column silently WIDENS. This view is INJECTED, -#: not applied, and it filters nothing — so the proportionate rule is the opposite one: take the -#: columns the table has, in this order, and skip the rest. An IG database that predates a -#: column simply shows the other four. -#: -#: ⚠ `location_guess` is SESSION B's item-16 column and may not exist yet. That is exactly why -#: this list is intersected rather than asserted: a hard requirement here would make the whole -#: view vanish (or the assembly refuse) on every tenant until B lands, and then appear by -#: surprise. `profile_url` closes the list as the click-through, which is what makes the view -#: usable rather than merely informative. -IG_OVERVIEW_COLUMNS = ('handle', 'full_name', 'followers', 'avg_engagement', - 'location_guess', 'enriched_at', 'profile_url') - -#: The id is PINNED, the `view_templates` discipline: re-assembling must update the same view -#: rather than mint "Overview 2". It also lets a user's own edits overlay it through the saved -#: -config loop below, exactly as a cohort projection does. -IG_OVERVIEW_ID = 'tpl_overview' - -#: How this function recognises an IG preset database WITHOUT importing the engine: two of the -#: profile preset columns is a stronger signal than any single one (a hand-made table could -#: plausibly own a column called `followers`; owning `followers` AND `avg_engagement` AND -#: `handle` is the preset set). `core/` must stay importable without the API layer, so this -#: mirrors `user_tables.PROFILE_PRESET_KEYS` the way that module mirrors the engine's. -_IG_SIGNATURE = ('handle', 'followers', 'avg_engagement') - - -def _ig_overview_view(fields, base): - """R8's curated Overview, or None when this table is not an Instagram one.""" - keys = {f['key'] for f in fields} - if not all(k in keys for k in _IG_SIGNATURE): - return None - visible = [k for k in IG_OVERVIEW_COLUMNS if k in keys] - return { - 'id': IG_OVERVIEW_ID, - 'name': 'Overview', - 'kind': 'system', - # NOT `locked`. The system view is locked because it is the identity of the table ("show - # me everything"); this one is a STARTING LAYOUT, and R8 calls it curated rather than - # fixed. A user who wants a sixth column should get one. - 'note': 'The five things worth seeing first on a creator. Sorted by reach.', - 'config': { - **dict(base), - 'visible': visible, - 'order': visible + [k for k in (f['key'] for f in fields) if k not in visible], - 'sorts': ([{'colId': 'followers', 'dir': 'desc'}] - if 'followers' in keys else []), - }, - } - - -def views_from_defs(defs, saved_views, fields, system_name="All customers", locked_lists=None, - view_order=None): - """Convert legacy list formulas into the shared serializable SavedView contract. - - `system_name` (wave 16 C-TOPIC) labels the system view per TOPIC ("All products" on the - product surface). The ID stays "all-customers" on every topic — the client pins it - (UNDELETABLE_VIEW_IDS, the landing default), and an id that varies by surface would fork - that contract for a label's sake. - - `locked_lists` (wave 17 R1) are the caller's cohorts, each PROJECTED as a saved view whose - id IS the cohort id — so every stored reference to that id (a `cohortLock` on another view, - an `is part of` condition, a folder placement) keeps pointing at the same thing and no - rewrite map is needed. Saved config OVERLAYS the projection through the same mechanism the - `list:` views have always used, which is what gives a locked view its own sort, filter, - columns and display mode with no new storage.""" - base = _default_view_config(fields) - views = [{ - "id": "all-customers", "name": system_name, "kind": "system", - "locked": True, "config": dict(base), - }] - # ⭐ WAVE-27 item 27 (R8) — the IG Overview, ABOVE All records. - # - # ⚠ INJECTED, not seeded into the store, and that is what makes "existing IG databases gain - # it too" true with no migration and no write on a read path. It is the same mechanism the - # system view above has always used; the pinned id means a user's own edits overlay it - # through the saved-config loop below rather than forking a second view. - _overview = _ig_overview_view(fields, base) - if _overview: - views.insert(0, _overview) - op_map = {">=": "gte", ">": "gt", "<=": "lte", "<": "lt", "=": "eq", - "contains": "contains"} - for name, definition in (defs or {}).items(): - filters = [] - for rule in definition.get("rules") or []: - if rule.get("field") not in {field["key"] for field in fields}: - continue - filters.append({ - "colId": rule["field"], - "op": op_map.get(rule.get("op"), "eq"), - "value": str(rule.get("value") if rule.get("value") is not None else ""), - }) - sort = str(definition.get("sort") or "") - sorts = ([{"colId": sort.lstrip("-"), - "dir": "desc" if sort.startswith("-") else "asc"}] - if sort.lstrip("-") in {field["key"] for field in fields} else []) - views.append({ - "id": "list:" + str(name), - "name": str(name), - "kind": "list", - "note": str(definition.get("note") or ""), - "config": { - **base, "filters": filters, "sorts": sorts, - "memberPids": [int(pid) for pid in definition.get("members") or [] - if isinstance(pid, int) or str(pid).isdigit()], - }, - }) - # Wave 17 R1 — the cohorts, as ordinary views. Appended BEFORE the saved-config overlay - # below so a user's own edits to a locked view (its sort, its columns, its display mode) - # land on the projection instead of creating a second row with the same id. - for _entry in (locked_lists or []): - if isinstance(_entry, dict) and _entry.get('id'): - views.append(locked_view_projection(_entry, base)) - index = {view["id"]: i for i, view in enumerate(views)} - for view_id, saved in (saved_views or {}).items(): - if not isinstance(saved, dict) or not isinstance(saved.get("config"), dict): - continue - clean = dict(saved) - clean["id"] = str(view_id) - if view_id in index: - views[index[view_id]] = clean - else: - views.append(clean) - # ⛔ WAVE 17 R1 — RE-STAMP THE LOCK AFTER THE OVERLAY. The loop above REPLACES a projected - # view with its saved record, and a saved record that omits `cohortLock` would hand back a - # view that shows the WHOLE BOOK under a locked view's name. That is not hypothetical: the - # client rebuilds `config` on every autosave (a column resize is enough), and the lock is - # identity here, not something the browser is the source of truth for. Read-side rather than - # write-side-only on purpose — this also repairs any record already written by another path. - _locked_ids = {e['id']: e for e in (locked_lists or []) - if isinstance(e, dict) and e.get('id')} - if _locked_ids: - for _v in views: - _entry = _locked_ids.get(_v.get('id')) - if not _entry: - continue - _v['kind'] = LOCKED_VIEW_KIND - _v['config'] = {**(_v.get('config') or {}), 'cohortLock': _v['id']} - # One thing, one name: the cohort store owns it (the rename event routes there), so - # a stale `name` on the saved record can never fork into a second title. - _v['name'] = _entry.get('name') or _v['id'] - # ── ⭐ WAVE-27 item 5, contract C7: the PER-USER VIEW ORDER ─────────────────────────────── - # - # `view_order` is a list of view ids this user dragged into place. Applied LAST, over the - # finished list, so it reorders whatever the assembly produced without having to know how any - # of it got there (system, list:, cohort projection, saved, injected Overview). - # - # ⛔ THE SYSTEM VIEW STAYS AT INDEX 0 (C7), and it is re-pinned here rather than trusted to - # sort correctly: `all-customers` is the client's landing default and one of its - # UNDELETABLE_VIEW_IDS, so a stored order that happened to omit it — or list it third — - # would move the rail's home row. ⚠ R8's Overview is the ONE thing allowed above it, because - # the owner put it there; it is re-pinned with the system view so a drag cannot bury it - # either. Both are facts about the table rather than the user's arrangement of it. - # - # ⚠ UNKNOWN IDS APPEND IN SERVER ORDER (C7). A view created since this order was stored, or - # one shared to this user yesterday, must APPEAR — dropping it would make sharing look - # broken, and the failure would be invisible to whoever shared it. Ids in the stored order - # that no longer resolve are simply skipped. - if view_order: - _rank = {vid: i for i, vid in enumerate(view_order) if isinstance(vid, str)} - _pinned = [v for v in views if v.get('id') in (IG_OVERVIEW_ID, 'all-customers')] - _rest = [v for v in views if v.get('id') not in (IG_OVERVIEW_ID, 'all-customers')] - # A stable sort over a rank that DEFAULTS TO THE END keeps unranked views in their - # server order behind the ranked ones, rather than interleaving them by accident. - _rest.sort(key=lambda v: _rank.get(v.get('id'), len(_rank) + 1)) - views = _pinned + _rest - return views - - -def workspace_wire(ws, uname, pool_pids, defs=None, scope_key='customer', storage_key=None, - fields_base=None, with_cohorts=True): - """The client's `GridWorkspace` WIRE SHAPE from the stored table workspace — the ONE - projection, shared by both servers (app.py's `_table_grid` and the API's `/workspace`). - - ⛔ WHY THIS EXISTS (2026-07-30). The API route used to return the STORE shape with no - `storageKey` — and the client validator (`fetchWorkspace`) requires one, so the standalone - shell silently discarded the whole workspace: saved views never rendered and `cohortMode` - never arrived (the Cohort route drew the Customer surface). Duplicating the host's inline - projection into the route would have re-created the same drift one wave later; extracting it - means the wire can only be one thing. - - Returns `(workspace, fields, views, cohort_lists)` — the extra three because the host - interleaves further work (docs, derived cells, measure sets) that consumes them. - - HOST-ONLY extras stay with the host: `docs`/`docPayload`, `pool`, `hideViews`, - `cohortMode`/`scopeChoice` (the API stamps its own from `?scope=`). - - Wave 16 C-TOPIC: `fields_base` selects the canonical contract (absent = customer, - byte-identical). - - ⭐ WAVE 19 / R9 — `with_cohorts` NO LONGER MEANS "customer only". Wave 16 set it False on the - product surface because cohorts were a single customer-keyed bucket, so resolving them against - product pids would have intersected two unrelated id spaces and printed a plausible, - meaningless member count. `modules.cohort` is scope-parameterized now: the lists come from - THIS topic's bucket (`cohort_mod.scoped(scope_key)`), so their ids are this topic's ids and - the intersection with `pool_pids` is the ordinary one. The flag survives as an honest OFF - switch for a surface that wants no membership channel at all — it is not a topic wall. - """ - import modules.cohort as cohort_mod - - cohort_lists = [] - if with_cohorts: - for cid, c in sorted(cohort_mod.scoped(scope_key).visible(uname, pool_pids).items(), - key=lambda kv: (kv[1].get('name') or '').lower()): - members = [p for p in (c.get('members') or []) if p in pool_pids] - entry = {'id': cid, 'name': c.get('name') or cid, 'pids': members} - # Rule 8b: a member can drop out of the 24-month pool without the cohort being - # wrong, and a silently smaller cohort is exactly what the unverifiable-count rule - # forbids. - missing = len(c.get('members') or []) - len(members) - if missing: - entry['missing'] = missing - cohort_lists.append(entry) - - fields = fields_from_workspace(ws, cohorts=bool(cohort_lists), scope_key=scope_key, - fields_base=fields_base) - views = views_from_defs(defs or {}, ws.get('views'), fields, - # Wave 21 (item 3, R6): the system view's name is TOPIC-DERIVED. A - # user database's default view used to read "All customers" — a - # compiled customer literal minted on every topic, one half of the - # owner's "my new database looks like RI's Customer table". The ID - # stays 'all-customers' everywhere (pinned client+server — the - # client's UNDELETABLE set and the view pin both name it). - system_name=("All products" if scope_key == 'product' - else "All records" - if str(scope_key or '').startswith('ut_') - else "All customers"), - locked_lists=cohort_lists, - # ⭐ WAVE-27 item 5 (C7) — this user's own rail arrangement, from - # their own stratum. Read here rather than sorted by the client so - # the ORDER a request answers with is the order that was stored: - # sorting client-side would make the rail settle after a paint on - # every load, and shared views would land in server order first. - view_order=ws.get('viewOrder')) - workspace = {'storageKey': storage_key, 'views': views, 'lists': cohort_lists} - # Owner item 3 (2026-07-31): where this user left off. The client's own localStorage copy - # wins when present; this is the server's answer for a FRESH browser, which used to fall - # all the way to the system default view (and whatever display mode was stored on it). - if ws.get('activeViewId'): - workspace['activeViewId'] = str(ws['activeViewId']) - - # FOLDERS (owner item 11, contract C4), re-validated at SERVE time: a view or cohort can be - # deleted by a path that knows nothing about folders, and the placement map must not outlive - # the thing it points at. - _folders = clean_folders(ws.get('folders')) - _view_ids = {v['id'] for v in (views or []) if isinstance(v, dict) and v.get('id')} - # ── WAVE 17 R1 (C-LOCKV amendment 2026-08-03): the two folder surfaces become ONE, AT - # SERVE TIME rather than by a store migration. A locked view is an ordinary view now, so - # its folder has to be an ordinary view folder — but rewriting the stored map would be a - # one-shot write that has to be got right once, while this is a projection that is right - # every time it runs. New drags write to `views` anyway (the client only knows that - # surface), so `cohorts` drains on its own and never needs a second pass. - # ⚠ A cohorts-surface folder whose id ALREADY names a views folder is DROPPED, not merged: - # re-parenting somebody's list into a folder that merely shares an id is a worse outcome - # than the list appearing at the root, where it is visible and one drag from home. - _cf = list(_folders.get('cohorts') or []) - if _cf: - _vf = list(_folders.get('views') or []) - _taken = {f['id'] for f in _vf} - _order = len(_vf) - for _f in _cf: - if _f['id'] in _taken: - continue - _vf.append({**_f, 'order': _order}) - _order += 1 - _folders['views'] = _vf - _raw_item_folders = dict(ws.get('itemFolders') or {}) - if _raw_item_folders.get('cohorts'): - # Cohort placements now describe VIEWS (same ids — that is the point of preserving them). - # A placement already stored on the views surface WINS: it is the more recent act. - _raw_item_folders['views'] = {**dict(_raw_item_folders.get('cohorts') or {}), - **dict(_raw_item_folders.get('views') or {})} - _placed = clean_item_folders( - _raw_item_folders, _folders, - {'views': _view_ids, 'cohorts': {c['id'] for c in cohort_lists}}) - if _folders.get('views'): - workspace['folders'] = _folders['views'] - # ⛔ `cohortFolders` IS NO LONGER EMITTED. The rail has no cohorts section to fold, and a - # wire that still described one would invite a second rendering of rows that are now views. - _vplaced = _placed.get('views') or {} - for _v in (views or []): - if isinstance(_v, dict) and _v.get('id') in _vplaced: - _v['folderId'] = _vplaced[_v['id']] - _cplaced = _placed.get('cohorts') or {} - for _c in cohort_lists: - if _c['id'] in _cplaced: - _c['folderId'] = _cplaced[_c['id']] - - # RECORD LAYOUT (wave 2026-08-02, C-LAYOUT): the per-user record-detail field order, - # re-validated at SERVE time exactly like folders — a field can be deleted by a path - # that knows nothing about this stratum, and a stale key must not outlive its field. - _rl = ws.get('recordLayout') - if isinstance(_rl, dict) and isinstance(_rl.get('order'), list): - _fkeys = {f['key'] for f in fields if isinstance(f, dict) and f.get('key')} - _order, _seen = [], set() - for _k in _rl['order'][:200]: - _k = str(_k or '') - if _k and _k in _fkeys and _k not in _seen: - _seen.add(_k) - _order.append(_k) - if _order: - workspace['recordLayout'] = {'order': _order} - - return workspace, fields, views, cohort_lists - - -def embed_html_path(): - """The first existing candidate path for the inlined single-file build, or None.""" - for p in _EMBED_CANDIDATES: - if p.is_file(): - return p - return None - - -def scope_counts(shown, matched, total): - """The honest 'N of M' a SERVER-WINDOWED table must carry (CG-2). - - `matched` and `total` MUST come from their own queries over the whole scope. Never pass - `len(rows)` as `matched` — that is the silent [:N] this exists to prevent: the page would - report the window size as though it were the result size. - - Refuses the shapes that could only be a mistake, because a wrong count here is invisible on - screen (it looks like a smaller dataset, not like an error). - """ - shown, matched, total = int(shown), int(matched), int(total) - if matched > total: - raise ValueError(f"matched ({matched}) exceeds total ({total}) — a filter cannot match " - f"more rows than the scope holds") - if shown > matched: - raise ValueError(f"shown ({shown}) exceeds matched ({matched}) — the window cannot hold " - f"more rows than the filter matched") - return {"shown": shown, "matched": matched, "total": total, "windowed": True} - - - -# ⛔ EXIT-6 (2026-08-04): `build_html`, `component_dir`, `render` and `_DECLARED_COMPONENTS` WERE -# HERE, and they are gone with Streamlit. They were the EMBED HOST — the path that declared the -# prebuilt bundle as a `streamlit.components.v1` custom component (or injected the single-file -# HTML build as a fallback) so the React grid could be drawn inside a Streamlit page. -# -# THIS MODULE ITSELF SURVIVES, and that distinction is the whole point: `aios_grid.py` is imported -# at 11 sites across `aios-web/api/` plus `harness/semantic.py` — it owns the canonical field -# contract, the workspace wire and the count envelope. Only the ~95 lines that knew about a HOST -# went; the rest never did. Its one and only `import streamlit` lived inside `render`, lazily, and -# left with it. `api/verify_no_streamlit.py` now gates that nothing here re-imports it. -# -# Deleted with them: `aios_grid_embed.html` + `aios_grid_component/index.html` (a 2.1 MB prebuilt -# bundle), `build_embed.py` that produced them, and `deploy_hf.py`'s embed-staleness guard. The -# React app is now served directly by the FastAPI container — there is no twin to keep fresh, so -# the entire class of "the code shipped but the bundle did not" is retired rather than guarded. +"""aios_grid — embed the AIOS React/glide Airtable-style grid inside Streamlit. + +This is REUSABLE MODULE INFRASTRUCTURE: the same self-contained grid that runs as the +standalone aios-web app is inlined into a single HTML file and hosted inside a Streamlit +component. The React bundle reads its data from `window.__AIOS_GRID__` (an object +`{fields, rows}`) that we inject into the page before the app's module script runs +— so there is NO backend and NO /api call in the Streamlit container; the browser only ever +sees the derived JSON we hand it. + +Usage (from any page): + import aios_grid + aios_grid.render(aios_grid.rows_from_pool(pool_rows), aios_grid.FIELDS) + +Design notes: + * ZERO app-internal imports (no `modules.*` / `core.*`). FIELDS is a literal and + `rows_from_pool` takes already-built rows — so this helper is tenant/module-agnostic and + sidesteps deploy_hf.py's import guard entirely. + * The built HTML ships as `aios_grid_embed.html` in THIS directory (added to + deploy_hf.py INCLUDE). Build it with `npm run build:embed` in aios-web/web, then copy + dist-embed/index.html -> platform/aios_grid_embed.html (see build_embed.py). + For LOCAL dev before that copy, we fall back to reading dist-embed/index.html directly. + * The preferred host is a Streamlit Components v1 bridge. It sends data/view/schema args + into the React app and returns guarded events for persistence in the tenant store. The old + injected-HTML path remains a read/local-write fallback when component assets are absent. +""" +import json +import math +import re +from pathlib import Path + +_HERE = Path(__file__).resolve().parent + +# Where the inlined single-file build lives. FIRST match wins: +# 1. the shipped copy in this dir (what deploy_hf.py uploads to the Space) +# 2. the raw build output in the sibling aios-web tree (local dev, pre-copy) +_EMBED_CANDIDATES = [ + _HERE / "aios_grid_embed.html", + _HERE.parent / "aios-web" / "web" / "dist-embed" / "index.html", +] +_COMPONENT_CANDIDATES = [ + _HERE / "aios_grid_component", + _HERE.parent / "aios-web" / "web" / "dist-embed", +] +_DECLARED_COMPONENTS = {} + +# --- the FIELD CONTRACT — loaded from the CANONICAL source `aios_grid_fields.json` in THIS +# directory (the SINGLE source of truth, shared with aios-web/api/main.py). source='odoo' is +# READ-ONLY; source='overlay' is the editable stratum (notes/tags) that lives OUTSIDE Odoo. +# +# Why a sibling JSON and NOT an import: aios_grid.py's contract is ZERO app-internal imports so +# it stays tenant/module-agnostic and sidesteps deploy_hf.py's import guard. A JSON next to the +# module preserves that exactly — no import, no guard interaction — while still single-sourcing +# the values (a build-time copy from another tree would reintroduce the drift we're removing). +# The file ships to the Space via deploy_hf.py INCLUDE. Edit the JSON, then run +# aios-web/verify_fields_contract.py. --- +_FIELDS_PATH = _HERE / "aios_grid_fields.json" + + +def _load_fields(): + if not _FIELDS_PATH.is_file(): + raise FileNotFoundError( + f"aios_grid: canonical field contract missing at {_FIELDS_PATH}. It is the single " + "source of truth for the grid schema and MUST ship (deploy_hf.py INCLUDE lists it)." + ) + doc = json.loads(_FIELDS_PATH.read_text(encoding="utf-8")) + fields = doc.get("fields") if isinstance(doc, dict) else doc + if not isinstance(fields, list) or not fields: + raise ValueError(f"aios_grid: {_FIELDS_PATH} has no 'fields' list.") + return fields + + +FIELDS = _load_fields() + + +def product_fields(): + """The PRODUCT table's canonical contract (wave 15 C-TOPIC, `product_data` key in the same + JSON). A separate accessor rather than a second module constant so the one file-read and the + one failure mode stay shared with `FIELDS`.""" + doc = json.loads(_FIELDS_PATH.read_text(encoding="utf-8")) + fields = (doc.get("product_data") or {}).get("fields") + if not isinstance(fields, list) or not fields: + raise ValueError(f"aios_grid: {_FIELDS_PATH} has no product_data.fields list.") + return fields + +# text/date fields pass through untouched; every OTHER odoo (numeric) field is rounded — +# mirrors aios-web/api/main.py _payload() exactly so embed == standalone byte-for-byte. + + +def _round(v): + return round(v) if isinstance(v, (int, float)) and not isinstance(v, bool) else v + + +def _round2(v): + """D-153 (W37, asked by lane B) — MONEY AND PERCENT KEEP THEIR DECIMALS. + + `_round`'s whole-number convention is right for counts and wrong for currency: `first_cost` + 3.47 shipped as `3`, so every money cell on every Odoo grid was a whole dollar. + + ⚠ `pct` IS AFFECTED TOO, and that is the half most likely to be dropped. `cells.ts` renders a + percent as `num(v).toFixed(1) + "%"`, so a `yoy_pct` of 12.34 was `round()`ed to 12 and painted + "12.0%" — one decimal of a number that HAS one, which reads as precision rather than as loss. + ⛔ NOT written as a `_round(v, nd=0)` default argument, deliberately: `round(v, 0)` returns a + FLOAT where `round(v)` returns an INT, so every integer column's wire shape would change under + an edit that looks purely cosmetic. + """ + return round(v, 2) if isinstance(v, (int, float)) and not isinstance(v, bool) else v + + +# Types a USER may create from the column menu (owner item 7, 2026-07-26). Mirrors +# customer-grid/types.ts CREATABLE_TYPES; verify_fields_contract.py holds the two in step. +# select — a single-select with its own `options` (the "Status" a user wants to add; distinct +# from the Odoo-sourced `status` lifecycle, which is not user-defined) +# user — an assignee, whose choices come from the HOST's real user list, never from here +#: `multiselect` (wave-2 item 5, 2026-07-27): the Airtable-style MULTI select — declared +#: options like `select`, but the cell holds a comma-joined SET and the row belongs to every +#: member of it (the `multi` grouping/cell contract the Cohorts column established). +#: Wave-5 item 11 (2026-07-27): `checkbox` (cell = bool; the overlay stores '1' or '') · +#: `phone` / `email` / `url` (text-family with per-type rendering) · `rating` (top-level +#: `max`, SVG stars client-side — never emoji) · `created_time` (READ-ONLY, renders the row's +#: `_created`) · `formula` (READ-ONLY, client-computed from the row's other cells). +#: ⭐ Wave-19 R7 / contract C5: `image` — a PICTURE on a record. The cell holds a string +#: REFERENCE, never bytes: a product `code`, an `ed:` editorial asset, or `rec:` for +#: something uploaded through the field (`POST /api/v1/assets/records`). Storing a reference is +#: what lets Royal's 1,142 existing SKU masters appear with nothing re-uploaded, and it keeps the +#: overlay stratum the size it is — base64 bytes in a JSON blob read on every render would be a +#: megabyte-per-row tax on the whole store. Editable by NATURE (deliberately NOT in +#: `READONLY_CUSTOM_TYPES`): the ref is what the upload endpoint hands back, and the client PATCHes +#: it through the ordinary overlay wall rather than the asset route writing cells behind it. +#: ⭐ WAVE 23 (C7) — `json` joined: a cell holding a whole DOCUMENT (an Instagram comment thread, +#: a webhook payload, a scraped blob) that opens in its own viewer instead of being flattened +#: into one unreadable line. It is EDITABLE by nature, like `image`: the value is still a plain +#: string on the wire, so it rides the ordinary overlay wall — what makes it a json field is that +#: `grid_events` REFUSES a write that does not parse (a column promising structure must not +#: silently hold something that isn't). +#: ⭐ 2026-08-07 — `link` and `rollup` JOINED (the relational wave). They ride here because +#: `UT_FIELD_TYPES` must stay a SUBSET of this set (gated in verify_api's W18-UT section) — the +#: surfaces that CREATE them are the user-table databases, where a relation between two tables is +#: a thing that exists. On the Odoo-backed Customer/Product grids there is no second user table to +#: point at, so the column menu there simply never offers one. +#: ⭐⭐ WAVE-34 (owner ruling R13) — `ai_enrich` JOINS, and it had to join HERE in the same change +#: that put it in `UT_FIELD_TYPES`, not a ticket later. The wave planned a SERVER-FIRST landing on +#: the reasoning that a kind the client does not offer is invisible while a kind the server refuses +#: deletes a column. That reasoning is sound and the conclusion was still wrong, because THREE +#: parity gates chain over these sets and none of them permits a partial landing: +#: `verify_api` W18-UT `UT_FIELD_TYPES - CUSTOM_FIELD_TYPES == set()` (this line) +#: `aw_fields_contract` §5 `CREATABLE_TYPES == CUSTOM_FIELD_TYPES`, EXACT set equality +#: `types.ts` `CREATABLE_TYPES: readonly FieldType[]`, so the union must carry it too +#: Measured live by lane B at 17:09: `api_api` went 969/969 to 968/969 the moment the kind entered +#: `core/user_tables.py` alone, printing `got {'ai_enrich'} want set()`. The comments on +#: `json`, `link`/`rollup` and `code` below all say the same thing in their own words. +CUSTOM_FIELD_TYPES = {"text", "select", "multiselect", "user", "int", "currency", "pct", "date", + "checkbox", "phone", "email", "url", "rating", "created_time", "formula", + "automation", "image", "json", "link", "rollup", "code", "ai_enrich"} + +#: ⭐ WAVE-27 item 13 (owner ruling R13) — the `code` field's LANGUAGES. +#: +#: R13 is explicit that this kind is "syntax-highlighted storage + language config, NO execution +#: engine". So a language is a RENDERING hint and nothing else: it selects a highlighter, it never +#: selects an interpreter, and no value here may ever grow a run path. The list is short on +#: purpose — every entry costs a highlighter the client actually has to implement, and an +#: unimplemented language would paint plain text under a label promising colour. +#: +#: `plain` is the default and the fallback, so it is never a second way to say nothing: it is the +#: honest answer for a snippet whose language the user has not chosen. +CODE_LANGUAGES = {"plain", "json", "sql", "python", "javascript", "typescript", + "html", "css", "markdown", "yaml", "xml", "shell"} + + +def _clean_code(raw): + """The `code` field's config bag -> `{'language': ...}`, or None. + + Deliberately OPTIONAL rather than required (the `automation` posture, not `link`'s): a code + column with no declared language is a legitimate state — it stores and highlights as plain + text — so refusing the FIELD over a missing bag would block the ordinary create path. An + unknown language falls back to `plain` rather than refusing, because the value is a rendering + hint: dropping the user's column to punish a typo in a highlighter name would be the + disproportionate half of the fail-closed rule. + + ⚠ `plain` RETURNS NONE, and that is what makes the control reversible rather than one-way. + Absent already means plain, so storing `{'language': 'plain'}` would be the default wearing a + second name (the `kanbanClamp` law). But the patch path resolves an OMITTED key to the + previous value — so if plain were merely omitted by the client, switching a column back from + SQL to Plain text would keep storing SQL and read as a control that does not save. Sending + the bag explicitly and having it evaluate to None here means: omit = keep, plain = clear. + """ + if not isinstance(raw, dict): + return None + lang = str(raw.get('language') or 'plain').strip().lower() + if lang not in CODE_LANGUAGES or lang == 'plain': + return None + return {'language': lang} +#: User-created types whose CELLS are read-only: their values are computed (formula — client +#: side, any error degrades to BLANK) or system-owned (created_time = the row's `_created`). +#: Emitted with the cohort column's read-only mechanism — `source: 'odoo'` + `derived` — so +#: the client never offers an editor and the host never accepts a cell write for them. +READONLY_CUSTOM_TYPES = {"created_time", "formula"} +MAX_FIELD_OPTIONS = 50 +MAX_FORMULA_LEN = 500 +#: `rating` bounds. Airtable caps at 10; below 2 a rating is a checkbox. +RATING_MAX_DEFAULT, RATING_MAX_MIN, RATING_MAX_MAX = 5, 2, 10 + +#: Key prefix of a FORMULA-MEASURE column (owner item 7, 2026-07-27): a user-created field that +#: IS a measure over a window — `Sales · the last 90 days` as a column. Mirrors the client's +#: `measure_` keys in CustomerGrid.createField. Distinct from `custom_` because the two strata +#: could not be more different: `custom_` is the EDITABLE overlay (user-typed values), while a +#: measure field is READ-ONLY and its values are computed by the host per render. +MEASURE_FIELD_PREFIX = "measure_" +#: The numeric types a measure can render as (semantic._FORMAT_TYPE's range). +MEASURE_FIELD_TYPES = {"currency", "int", "pct"} + + +def _clean_options(raw): + """Choices for a `select`: strings, trimmed, de-duplicated case-insensitively, capped. + Mirrors types.ts parseOptions — a choice list that means one thing in the picker and + another in the filter dropdown is a column with two vocabularies.""" + out, seen = [], set() + for v in list(raw or [])[:MAX_FIELD_OPTIONS * 2]: + if not isinstance(v, (str, int, float)) or isinstance(v, bool): + continue + s = str(v).strip()[:120] + if not s or s.lower() in seen: + continue + seen.add(s.lower()) + out.append(s) + return out[:MAX_FIELD_OPTIONS] + + +def _clean_option_colors(raw, options): + """Choice-label -> #RRGGBB, limited to the field's canonical option vocabulary.""" + if not isinstance(raw, dict): + return {} + supplied = {} + for label, color in raw.items(): + if not isinstance(label, str) or not isinstance(color, str): + continue + clean = color.strip().upper() + if re.fullmatch(r"#[0-9A-F]{6}", clean): + supplied[label.strip().lower()] = clean + out = {} + for option in options or []: + color = supplied.get(str(option).strip().lower()) + if color: + out[str(option)] = color + return out + + +def _choice_appearance(raw, options): + """Validated select-family appearance. Absent colour toggle means legacy-on.""" + if not isinstance(raw, dict): + return {} + out = {} + if isinstance(raw.get("colorCodeOptions"), bool): + out["colorCodeOptions"] = raw["colorCodeOptions"] + colors = _clean_option_colors(raw.get("optionColors"), options) + if colors: + out["optionColors"] = colors + return out + + +def _clean_rating_max(raw): + """A rating's star count, bounded. Anything unusable is the default, not a refusal — the + field still holds its 1..max integers either way.""" + try: + return max(RATING_MAX_MIN, min(int(raw), RATING_MAX_MAX)) + except (TypeError, ValueError): + return RATING_MAX_DEFAULT + + +#: The field types a number-style display format may apply to. `formula` is here because its +#: RESULT is a number the client renders; `pct` already renders in points and takes decimals. +_NUMBER_FORMAT_TYPES = {"int", "currency", "pct", "formula"} + + +def _clean_format(raw, ftype): + """Per-type DISPLAY format (wave-5 item 10), fail-closed: unknown keys are DROPPED, wrong + types return None (the property is simply absent). Rendering-only — a format can change how + a value reads, never what it is, which is why this needs no parity gate of its own.""" + if not isinstance(raw, dict): + return None + out = {} + if ftype in _NUMBER_FORMAT_TYPES: + if isinstance(raw.get("thousands"), bool): + out["thousands"] = raw["thousands"] + if raw.get("decimals") is not None: + try: + d = int(raw["decimals"]) + except (TypeError, ValueError): + d = None + if d is not None and 0 <= d <= 4: + out["decimals"] = d + if isinstance(raw.get("abbrev"), bool): + out["abbrev"] = raw["abbrev"] + elif ftype in ("date", "created_time"): + if isinstance(raw.get("time"), bool): + out["time"] = raw["time"] + if raw.get("tz") in ("local", "utc"): + out["tz"] = raw["tz"] + return out or None + + +def _clean_permissions(raw): + """`{edit: 'everyone' | 'creator' | 'admins'}` or None (wave-5 item 1). WHO may set it is + the host handler's business (creator/admin, enforced fail-closed there); this validates + only the shape, like every other property here.""" + if isinstance(raw, dict) and raw.get("edit") in ("everyone", "creator", "admins"): + return {"edit": raw["edit"]} + return None + + +#: ⛔ THE CHARSET MUST ADMIT EVERY TOKEN THE CLIENT ENGINE PARSES, or a legal formula is +#: refused by a wall that is supposed to be structural (2026-08-03). +#: +#: This regex was written when a formula was arithmetic over refs. On 2026-07-31 the client +#: engine (owner item 2) gained STRING literals, `&` concatenation and `^` — CONCATENATE, TEXT, +#: LEFT/RIGHT/MID, and any `IF(cond, "yes", "no")`. This list was never widened to match, so +#: every such formula died here: `field_upsert` refused the create, and `fields_from_workspace` +#: dropped the column on read. Nothing went red — a refused create looks like a quiet failure +#: and a dropped column looks like a column nobody made. +#: +#: Found by trying to ship the owner's own Buy signal formula, which is `IF(..., "Buy now", +#: "OK")` and could not be created through the product UI at all. +#: +#: ⚠ IT IS STILL STRUCTURAL, and deliberately not a second grammar — that is the filter_sql-class +#: drift risk this file's docstring names. The charset says which characters may appear; the +#: engine says what they mean. `_quotes_balanced` below is the one structural rule the quote +#: character brings with it. +_FORMULA_CHARS = re.compile(r"^[\w\s{}()+\-*/.,<>=!^&\"]*$") +_FORMULA_REF = re.compile(r"\{([^{}]*)\}") + + +def _quotes_balanced(s): + """An even number of `"` — the structural half of string support. + + Sound because the engine's own escape is Excel's: `""` inside a string is one quote, and it + contributes TWO characters. So a well-formed expression always has an even count and an + unterminated string always has an odd one. What a balanced pair MEANS is the engine's + business, exactly as with parentheses. + """ + return s.count('"') % 2 == 0 + + +#: Wave-18 C5-AUTOFIELD. `kind` is a whitelist because an unknown kind would be a column that +#: silently never runs; `source` names where the run's subject comes from. +AUTOMATION_KINDS = {"instagram_profile"} +AUTOMATION_SOURCES = {"record_url_field", "self"} +MAX_AUTOMATION_SETTINGS = 12 + + +def _clean_automation(raw, valid_keys=None, flow_ids=None): + """Validate an `automation` config bag. Returns the clean dict, or None when there is + nothing valid to store (the column then renders as unconfigured — never invented). + + ⛔ WAVE 22 (contract C8, owner item 5) — NO FIELD WITHOUT A FLOW. With `flow_ids` given + (the WRITE path: grid_events / user_tables pass the tenant's automation-definition ids), + the bag MUST carry a `flowId` naming one of them — absent or naming a deleted flow is + refused, the same fail-closed direction as a formula ref that names no field. With + `flow_ids=None` (the READ path, `fields_from_workspace`) the law is NOT applied: a column + stored before the law must keep projecting — enforcement at read time would vaporise it, + which is the `_clean_formula` write/read split exactly. + """ + if not isinstance(raw, dict): + return None + kind = str(raw.get('kind') or '').strip() + if kind not in AUTOMATION_KINDS: + return None + source = str(raw.get('source') or 'record_url_field').strip() + if source not in AUTOMATION_SOURCES: + source = 'record_url_field' + out = {'kind': kind, 'source': source} + flow = str(raw.get('flowId') or '').strip()[:40] + if flow_ids is not None and (not flow or flow not in flow_ids): + return None + if flow: + out['flowId'] = flow + url_field = str(raw.get('urlField') or '').strip()[:80] + # fail closed on a ref that does not exist, exactly as _clean_formula does at WRITE time + if url_field and (valid_keys is None or url_field in valid_keys): + out['urlField'] = url_field + settings = {} + for k, v in list((raw.get('settings') or {}).items())[:MAX_AUTOMATION_SETTINGS]: + if isinstance(v, bool) or isinstance(v, (int, float)): + settings[str(k)[:40]] = v + elif isinstance(v, str): + settings[str(k)[:40]] = v[:200] + if settings: + out['settings'] = settings + return out + + +def _clean_formula(raw, valid_keys=None): + """STRUCTURAL passthrough for a formula field's expression (wave-5 item 9). + + Meaning is NOT checked here: the CLIENT engine owns the grammar (arithmetic over `{field}` + refs, ABS/ROUND/MIN/MAX/IF), and any evaluation error degrades to a BLANK cell — never a + wrong number. That is the `_clean_window` split one stratum up, and deliberately NOT a + Python mirror of the grammar: a second engine is the filter_sql-class drift risk. + Structure IS checked — charset, length, balanced parens, BALANCED QUOTES, well-formed + non-empty `{refs}` — and at WRITE time (`valid_keys` given) every ref must name a field this + table has, fail closed. At READ time refs are left alone: a referenced field deleted later + must blank the CELLS, not vaporise the column. + + ⚠ A `{ref}` INSIDE A STRING LITERAL is still checked against `valid_keys` at write time, so + `IF(x, "see {notafield}", "")` is refused. That is a false rejection and it is the + fail-closed direction: the alternative is teaching this function where strings begin and + end, which is the second grammar the paragraph above refuses to write. + """ + if not isinstance(raw, str): + return None + s = raw.strip() + if not s or len(s) > MAX_FORMULA_LEN or not _FORMULA_CHARS.match(s): + return None + depth = 0 + for ch in s: + if ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + if depth < 0: + return None + if depth: + return None + if not _quotes_balanced(s): + return None + refs = _FORMULA_REF.findall(s) + leftover = _FORMULA_REF.sub("", s) + if "{" in leftover or "}" in leftover: # unbalanced / nested braces + return None + if any(not r.strip() for r in refs): # a `{}` ref names nothing + return None + if valid_keys is not None and any(r not in valid_keys for r in refs): + return None + return s + + +def _clean_geocode(raw): + """One `geocode` bag → the stored shape, or None (dropped). + + ⭐⭐ W37-T22 (owner item 15 / ruling R4, contract C7). THE ONE validator for the geocode + pseudo-kind, called by BOTH field doors: `_field_extras` here (the Odoo grids) and + `core/user_tables._clean_field` via `_ag_geocode` (every `ut_*` user database). + + ⛔ IT WAS INLINE IN `_field_extras` UNTIL 2026-08-19 AND THAT IS WHY IT SHIPPED HALF-DONE. + `user_tables._clean_field` is a strict allowlist that returns only the bags it NAMES, so a + geocode column created on a user database was stored WITHOUT its `{addressField}` — the + *"created, named, configured, gone"* failure the `image` type shipped as in wave 19. Extracting + it rather than copying the rules is [[one-question-two-normalizers]]: a second copy is how the + two doors come to disagree about what a valid bag is. + + ⛔ WHOLE-KEY DROP (the `swipe` construction): a geocode column with no address column is not + a degraded column, it is a column no run can ever fill, and the honest state for that is the + unconfigured one the client refuses at `canCreate`. + + ⛔ THE FIELD'S `type` IS `text` AND THERE IS NO `geocode` FieldType. The kind is a PSEUDO-KIND + in the create picker (the `measure` construction), so `CUSTOM_FIELD_TYPES`, `UT_FIELD_TYPES`, + `CREATABLE_TYPES`, the shape/label tables and the cell renderer are all untouched — three + parity gates chain over those sets and none permits a partial landing. This is also why the + bag takes `format`'s posture and NOT `formula`'s: there is no type to pair it with, so an + absent bag is an ordinary text column rather than a refusal. + + ⚠ `addressField` is NOT checked against the table's keys: neither door is given `valid_keys`, + and the split this module already draws puts that loss on the surface that RENDERS. The + enricher must SAY the address column is gone rather than geocode an empty string. + """ + if not isinstance(raw, dict): + return None + addr = raw.get("addressField") + if not isinstance(addr, str) or not addr.strip(): + return None + out = {"addressField": addr.strip()[:80]} + # D-3 — the optional ISO 3166-1 alpha-2 hint, mapped straight onto the geocoder's + # `countrycodes` parameter. NORMALISED to upper case and stored only when it is exactly two + # letters: absent means "anywhere", and `""` would be that state's second spelling. A + # three-letter or numeric code is REFUSED rather than truncated, because a truncated code is + # a valid code for a different country. + cc = raw.get("country") + if isinstance(cc, str) and re.fullmatch(r"[A-Za-z]{2}", cc.strip() or ""): + out["country"] = cc.strip().upper() + return out + + +def _field_extras(saved, ftype): + """createdBy / permissions / format / scope — the validated passthrough the created strata + share (wave 5). `createdBy` is only ever WRITTEN host-side (the handler stamps it); here it + is carried so the client can gate its menus and the handler can enforce against it. + `scope` (wave-6 item 9): 'cohort' marks a field created as cohort-specific — the handler + stamps it at create (cohort page only) and preserves it like createdBy; carried here so the + client can label the field, filtered OUT of other pages by fields_from_workspace.""" + out = {} + who = saved.get("createdBy") + if isinstance(who, str) and who.strip(): + out["createdBy"] = who.strip()[:80] + perms = _clean_permissions(saved.get("permissions")) + if perms: + out["permissions"] = perms + fmt = _clean_format(saved.get("format"), ftype) + if fmt: + out["format"] = fmt + if saved.get("scope") == "cohort": + out["scope"] = "cohort" + # ── ⭐⭐ W37-T22 (owner item 15 / ruling R4, contract C7): THE GEOCODE COLUMN'S CONFIG ── + # + # `{addressField}` — which column this one reads. WHOLE-KEY drop (the `swipe` construction): + # a geocode column with no address column is not a degraded one, it is a column no run can + # ever fill, and the honest state for that is the unconfigured one the client refuses at + # `canCreate`. + # + # ⛔ THE FIELD'S `type` IS `text` AND THERE IS NO `geocode` FieldType. The kind is a PSEUDO-KIND + # in the create picker (the `measure` construction), so `CUSTOM_FIELD_TYPES`, `UT_FIELD_TYPES`, + # `CREATABLE_TYPES`, the shape/label tables and the cell renderer are all untouched — three + # parity gates chain over those sets and none permits a partial landing. + # ⚠ `addressField` is NOT checked against this table's keys: `_field_extras` is not given + # `valid_keys`, and the split this module already draws puts that loss on the surface that + # RENDERS ("what each mode MEANS ... is the client's business"). The enricher must SAY the + # address column is gone rather than geocode an empty string. + # ⚠ EXTRACTED to `_clean_geocode` 2026-08-19 (/validate-wave, W37-T22 clause 1). The rules + # did not change; what changed is that `core/user_tables.py` now reaches THIS function through + # `_ag_geocode` instead of carrying a second copy. See `_clean_geocode`'s own note. + geo = _clean_geocode(saved.get("geocode")) + if geo: + out["geocode"] = geo + corrected_from = saved.get("labelCorrectedFrom") + correction_id = saved.get("labelCorrectionId") + if (isinstance(corrected_from, str) and corrected_from.strip() + and isinstance(correction_id, str) and correction_id.strip()): + out["labelCorrectedFrom"] = corrected_from.strip()[:120] + out["labelCorrectionId"] = correction_id.strip()[:180] + return out + + +#: The DERIVED column listing the cohorts a customer is in (owner, 2026-07-27). +#: +#: NOT in `aios_grid_fields.json`, deliberately. That contract is per-TABLE and shared with the +#: standalone API and the dev sample; a cohort is per-USER, so the column exists exactly when the +#: caller has cohorts — the same condition under which the `__cohort__` FILTER field is offered. +#: Putting it in the canonical contract would mean an always-present column that is empty for +#: everyone else, plus three consumers to keep in step for a value none of them can produce. +COHORT_COLUMN = 'cohorts' + + +def cohort_field(label='Locked views'): + """The derived membership column's descriptor. + + ⚠ WAVE 17 item 14 / C-STR — THE LABEL AND THE NOTE SPEAK THE NEW VOCABULARY; THE KEY DOES + NOT. `COHORT_COLUMN` is still `'cohorts'` and the function is still `cohort_field`, because + every stored view that shows or groups by this column names it by KEY, and every gate in + two runtimes names the function. The owner renamed a CONCEPT ("we should stop calling it + Cohort, but locked instead"), which is a change to what a reader sees — renaming the + identifiers would break saved views to change a word nobody reads. + + `source: 'odoo'` is doing ONE job here and it is not provenance: the client keys editability + off `source == 'overlay'`, and `_cl_handle_grid_event` accepts cell writes only for overlay + keys. So 'odoo' is what makes this column READ-ONLY at both ends. `derived: True` is what + stops the column menu calling it a "source field" on that basis. + + ⚠ `filterable: False`, and the replacement is the `Where [Cohort] […]` CONDITION, not another + column. Text ops over a joined string would ALMOST work and disagree at the edges — `contains + "VIP"` also matches a cohort called "VIP club" — and a filter that is nearly right is worse + than one that is absent. + """ + return { + 'key': COHORT_COLUMN, 'label': label, 'type': 'text', 'source': 'odoo', + 'default': False, 'filterable': False, 'derived': True, 'multi': True, + 'note': 'The locked views this customer is in, newest first. A locked view holds a SET, ' + 'so grouping by this column puts a customer under EVERY view they are in — the ' + 'group counts therefore add up to more than the record count, which stays the ' + 'number of distinct customers. Read-only: membership changes only by adding or ' + 'removing customers on the locked view itself.', + } + + +def cohort_cells(cohorts, allowed_pids=None): + """`{pid: {'cohorts': 'Q3 calls, Lost'}}` from `[{id,name,pids}, ...]`. + + Built per render from the caller's OWN cohorts and handed to `rows_from_pool` as `derived`, + never merged into the cached pool rows — those are shared across users, and stamping one + user's cohorts onto them would leak the membership to everybody else on the next render. + """ + cells = {} + for c in cohorts or []: + # ⚠ The COMMA is the separator the client splits on to group a customer into EVERY + # cohort they are in, so it cannot also occur inside a name. Cohort names are free text + # ("Q3 calls, west" is a name somebody will type), so a comma is replaced here rather + # than left to break the split silently — one group called "Q3 calls" and another called + # "west" would be two lists that do not exist. The cost is cosmetic and confined to the + # cell; the Cohort page still shows the name the user typed. + name = str(c.get('name') or c.get('id') or '').replace(',', ' ').strip() + for pid in c.get('pids') or (): + if allowed_pids is not None and pid not in allowed_pids: + continue + cells.setdefault(pid, []).append(name) + return {pid: {COHORT_COLUMN: ', '.join(names)} for pid, names in cells.items()} + + +def clean_measure_field(raw, offered): + """Validate one UNTRUSTED formula-measure field (owner item 7) against the caller's OFFER. + + `offered` is `{measure key: {label, type}}` from `measure_filter.measure_fields(team_id)` — + the same admission the condition builder uses, so a field can only name a measure this + caller could also filter by. Returns the canonical stored shape, or None (fail closed). + + `source:'odoo'` + `derived:True` is the cohort column's read-only mechanism, reused: + the client keys editability off `source == 'overlay'` and the host accepts cell writes only + for overlay keys, so a measure column cannot be typed into at either end. `filterable:False` + because the REPLACEMENT is the measure CONDITION with the same measure and window — the + governed, gate-proved path (CG-8/CG-12) — not text ops over a derived cell. + """ + if not isinstance(raw, dict): + return None + key = str(raw.get("key") or "") + if not key.startswith(MEASURE_FIELD_PREFIX) or len(key) > 80: + return None + spec = raw.get("measure") + if not isinstance(spec, dict): + return None + m = (offered or {}).get(spec.get("key")) + if not m: + return None # not admitted for this caller -> fail closed + window = _clean_window(spec.get("window")) + if window is None: + return None # a measure column with no period is not a column + mtype = m.get("type") if m.get("type") in MEASURE_FIELD_TYPES else "currency" + return { + "key": key, + "label": str(raw.get("label") or m.get("label") or "Measure")[:120], + "type": mtype, + "source": "odoo", + "default": True, + "custom": True, + "derived": True, + "filterable": False, + "agg": "sum" if mtype in ("currency", "int") else None, + "note": str(raw.get("note") or "")[:2000], + "measure": {"key": str(spec.get("key"))[:80], "window": window}, + } + + +def measure_fields_of(fields): + """The formula-measure columns among `fields` — the ones whose values the host must compute + per render (see `rows_from_pool`'s `derived`).""" + return [f for f in fields or [] if isinstance(f.get("measure"), dict)] + + +def fields_from_workspace(workspace=None, cohorts=False, scope_key=None, fields_base=None): + """Overlay persisted notes/custom fields onto the immutable source-field contract. + + `cohorts=True` appends the derived cohort column — see `cohort_field`. Appended LAST and + `default: False`, so it never displaces a column somebody already reads; the Fields menu is + where you turn it on. + + Three saved strata pass through: notes on base fields, `custom_` overlay fields (editable), + and `measure_` formula-measure fields (read-only, host-computed — owner item 7). A measure + field was validated against the caller's measure OFFER when it was written + (`clean_measure_field`); here only its SHAPE is re-checked, because this module has zero + app-internal imports and cannot know the offer. A measure that has since become + unanswerable (a BU scope on a company-level measure) degrades to a BLANK column at value + time, never to an error. + + `scope_key` (wave-6 item 9) names the PAGE doing the asking ('customer' / 'cohort'). A + saved def carrying `scope` is emitted only when it matches — so a cohort-specific field + never appears on the Customer table. FAIL-CLOSED: a caller that passes no scope_key sees + only unscoped (global) fields; base contract fields are never scoped. + + `fields_base` (wave 16 C-TOPIC) — the canonical contract to overlay onto. Absent = the + CUSTOMER contract (`FIELDS`), byte-identical to before the parameter existed; the product + surface passes `product_fields()`. The workspace dict a caller hands in must already be the + matching table object's bucket — this function cannot tell a customer overlay from a + product one, which is exactly why the buckets are separate stores. + """ + saved = dict((workspace or {}).get("fields") or {}) + out = [] + base_fields = fields_base if fields_base is not None else FIELDS + base_keys = {field["key"] for field in base_fields} + for base in base_fields: + meta = saved.get(base["key"]) or {} + field = dict(base) + if isinstance(meta.get("note"), str): + field["note"] = meta["note"][:2000] + # Wave-5 item 10: a saved DISPLAY format on any base field (a preset included) — how a + # number or date READS, per user. Rendering-only, so this is the whole acceptance. + fmt = _clean_format(meta.get("format"), base.get("type")) + if fmt: + field["format"] = fmt + # ⭐ W29-T83 — the saved COLUMN SUMMARY, the read half of the write door in + # `grid_events.field_upsert`. Without this the value round-trips into the store and is + # never served back, which looks exactly like a write that never happened + # ([[read-path-cannot-witness-write-path]]). Absent = whatever the contract declares. + if meta.get("agg") in FIELD_AGGS: + field["agg"] = meta["agg"] + # PRESET measure fields (wave-2 item 8): a preset+measure base may take a saved + # window/label override. Wave 6 deleted every preset member (the owner's + # no-buildable-presets rule) so this branch is currently MEMBERLESS — kept as the + # measure_ path's twin for any future preset-carrying contract, and because deleting + # it would silently change what a re-added preset means. + if base.get("preset") and isinstance(base.get("measure"), dict): + saved_measure = meta.get("measure") if isinstance(meta.get("measure"), dict) else {} + window = _clean_window(saved_measure.get("window")) + if window is not None: + field["measure"] = {"key": base["measure"]["key"], "window": window} + if isinstance(meta.get("label"), str) and meta["label"].strip(): + field["label"] = meta["label"][:120] + out.append(field) + for key, field in saved.items(): + if key in base_keys or not isinstance(field, dict): + continue + if field.get("scope") and field.get("scope") != scope_key: + continue # a cohort-specific field on another page (wave-6 item 9) + if (str(key).startswith(MEASURE_FIELD_PREFIX) + and isinstance(field.get("measure"), dict)): + window = _clean_window(field["measure"].get("window")) + mkey = str(field["measure"].get("key") or "") + if window is None or not mkey: + continue + mtype = field.get("type") if field.get("type") in MEASURE_FIELD_TYPES else "currency" + out.append({ + "key": str(key)[:80], + "label": str(field.get("label") or "Measure")[:120], + "type": mtype, + "source": "odoo", + "default": bool(field.get("default", True)), + "custom": True, + "derived": True, + "filterable": False, + "agg": "sum" if mtype in ("currency", "int") else None, + "note": str(field.get("note") or "")[:2000], + "measure": {"key": mkey[:80], "window": window}, + **_field_extras(field, mtype), + }) + continue + if not str(key).startswith("custom_"): + continue + ftype = field.get("type") + if ftype not in CUSTOM_FIELD_TYPES: + continue + if ftype in READONLY_CUSTOM_TYPES: + # Wave-5 items 9/11: the read-only user-created pair. Emitted with the cohort + # column's mechanism (source 'odoo' + derived) so the client never offers an + # editor and the host's overlay-write guard excludes them by construction. + # FILTERABLE since wave 6 (owner item 6): their values live client-side + # (formula computes over the row, created_time renders `_created`), this + # table's counts are client-mode, and the windowed count path never sees + # these tables — so the client engine answers them soundly. + entry = { + "key": str(key)[:80], + "label": str(field.get("label") or "Untitled")[:120], + "type": ftype, + "source": "odoo", + "derived": True, + "filterable": True, + "default": bool(field.get("default", True)), + "custom": True, + "note": str(field.get("note") or "")[:2000], + **_field_extras(field, ftype), + } + if ftype == "formula": + formula = _clean_formula(field.get("formula")) + if formula is None: + continue # a formula field without a formula is nothing + entry["formula"] = formula + out.append(entry) + continue + if field.get("source") != "overlay": + continue + options = (_clean_options(field.get("options")) + if ftype in ("select", "multiselect") else []) + if ftype in ("select", "multiselect") and not options: + # A select with no surviving choices can never hold a value. Dropping the COLUMN + # would lose the user's data; degrading it to text keeps every stored value + # readable and lets them re-add choices. + ftype = "text" + out.append({ + "key": str(key)[:80], + "label": str(field.get("label") or "Untitled")[:120], + "type": ftype, + "source": "overlay", + "default": bool(field.get("default", True)), + "custom": True, + # WAVE-29 C7: the whole vocabulary, not the `{"sum"}` literal that was here — a + # picker offering Average against a projection that only passes Sum through is the + # silent half of this feature. + "agg": field.get("agg") if field.get("agg") in FIELD_AGGS else None, + "note": str(field.get("note") or "")[:2000], + **({"options": options} if ftype in ("select", "multiselect") else {}), + **(_choice_appearance(field, options) + if ftype in ("select", "multiselect") else {}), + # comma-joined SET semantics (the Cohorts column's contract): the row belongs to + # every member, groups count it under each, the toolbar count stays distinct. + **({"multi": True} if ftype == "multiselect" else {}), + **({"max": _clean_rating_max(field.get("max"))} if ftype == "rating" else {}), + **({"automation": _clean_automation(field.get("automation"))} + if ftype == "automation" and _clean_automation(field.get("automation")) else {}), + # ⭐ WAVE-27 item 13 (R13) — the code column's language rides the wire, because the + # highlighter is chosen per column and the client cannot infer a language from a + # string. Absent = `plain`, which is what an unconfigured code column renders as. + **({"code": _clean_code(field.get("code"))} + if ftype == "code" and _clean_code(field.get("code")) else {}), + **_field_extras(field, ftype), + }) + if cohorts and not any(f.get('key') == COHORT_COLUMN for f in out): + # ⚠ The emptiness check is WAVE 19's, and it is about the topics R9 opened this column to. + # A user table's field keys are slugged from whatever its creator typed, so a column + # literally called "Cohorts" produces the key `cohorts` — and appending here unguarded + # would put TWO fields with one key on the wire. The client indexes fields by key, so the + # duplicate does not error: it silently paints one column's values under the other's + # header. The user's own column wins; the derived one steps aside rather than shadowing it. + out.append(cohort_field()) + return out + + +def rows_from_pool(pool_rows, fields=None, overlays=None, derived=None): + """Map customer_data.pool() dicts -> the API row shape the grid expects: + pid + each Odoo field (numeric fields rounded, text/date passed through) + the + persisted external overlay. Mirrors the standalone API payload contract. + + `derived` is `{pid: {key: value}}` for columns the HOST computes per render rather than + reads off the pool row — today just the cohort column. A separate argument from `overlays` + on purpose: `overlays` is the PERSISTED user stratum, and putting a value there that is + never written back would make the dict mean two things. + """ + fields = fields or FIELDS + odoo_fields = [field for field in fields if field["source"] == "odoo"] + overlay_fields = [field for field in fields if field["source"] == "overlay"] + derived_keys = [field["key"] for field in fields if field.get("derived")] + overlays = overlays or {} + derived = derived or {} + out = [] + for r in pool_rows: + pid = r.get("pid") + # `_created` (wave-5 item 11) rides every row like `pid` does — the datum the + # `created_time` field type renders, regardless of that field's own key. Not a Field: + # it has no column of its own until a user creates one. `lat`/`lon` (wave-7 W11) ride + # the same way: the Map VIEW's data, nullable, deliberately not a column. + row = {"pid": pid, "_created": r.get("_created") or "", + "lat": r.get("lat"), "lon": r.get("lon")} + for field in odoo_fields: + k = field["key"] + if field.get("derived"): + continue # not on the pool row — filled from `derived` below + v = r.get(k) + # D-153 — three roundings, not two: text/status/date pass through verbatim, currency + # and pct keep two decimals (`_round2`), everything else stays a whole number. + row[k] = v if field["type"] in {"text", "status", "date"} else ( + _round2(v) if field["type"] in {"currency", "pct"} else _round(v)) + saved = overlays.get(str(pid), {}) or {} + for field in overlay_fields: + row[field["key"]] = saved.get(field["key"], "") + got = derived.get(pid) or {} + for k in derived_keys: + # '' not None: a customer in no cohort has an EMPTY cohort list, and `is empty` on a + # text column is the question somebody will ask of it. + row[k] = got.get(k, "") + out.append(row) + return out + + +# --- the FILTER-TREE contract (mirrors customer-grid/types.ts) --------------- +# Ops the Airtable-parity condition builder can emit. isEmpty/isNotEmpty are +# VALUE-FREE (they legitimately carry no value and must never be dropped for it). +FILTER_OPS = {'contains', 'doesNotContain', 'eq', 'neq', 'isEmpty', 'isNotEmpty', + 'gt', 'gte', 'lt', 'lte', 'between', 'within', + # Wave 2026-08-02 (C-OPS): RANK operators — evaluated as a SET pass over the + # sibling-filtered domain by the client engine (useVisibleRows). The validator + # accepts them like any op (structural, not semantic); filter_sql REFUSES to + # compile them to row SQL (a per-row WHERE cannot express Top-N). aboveAvg / + # belowAvg are VALUE-FREE; the rest encode their argument in `value` as a + # string int (topN/bottomN 1..10000, inTopPct/inBottomPct 1..100, + # inQuartile 1..4, inDecile 1..10). Deliberately NOT in MEASURE_OPS: on a + # measure-carrying column they rank the field's own derived values. + 'topN', 'bottomN', 'inTopPct', 'inBottomPct', + 'aboveAvg', 'belowAvg', 'inQuartile', 'inDecile'} +#: The RESERVED pseudo-column of a cohort-membership leaf (owner item 5, 2026-07-26). It is not +#: a Field and never will be — a cohort is a hand-curated SET, so making it a column would mean +#: a cell per row per cohort. Mirrors customer-grid/types.ts COHORT_FIELD. +COHORT_FIELD = '__cohort__' +#: Ops a cohort leaf may carry (owner, 2026-07-27): set operators over a SET of cohorts. +#: Anything else is dropped. Deliberately DISJOINT from FILTER_OPS — see types.ts COHORT_OPS: +#: a set op reaching a column leaf would fall through the client engine's switch to "no +#: narrowing", so keeping the vocabularies apart makes the existing fail-closed drop do the work. +COHORT_OPS = {'anyOf', 'allOf', 'noneOf'} + +#: ⭐⭐ WAVE-37 T26 (owner item 14, contract C4) — THE VIEW-MEMBERSHIP LEAF's reserved +#: pseudo-column and its operators. Mirrors `customer-grid/types.ts` VIEW_FIELD / VIEW_OPS. +#: +#: ⛔ DISJOINT FROM `FILTER_OPS`, for the reason COHORT_OPS is: `matchFilter` answers an operator +#: it does not know by NOT NARROWING, so an op that reached a column leaf would widen the result +#: under an authoritative count. Keeping the two vocabularies apart means a column leaf carrying +#: `inView` is dropped by the same fail-closed path that drops `dropTable`. +#: ⚠ The leaf names exactly ONE view (singular where a cohort leaf names a set), so there is no +#: all-or-nothing rule to get wrong here — the id either resolves or the leaf goes. +VIEW_FIELD = '__view__' +VIEW_OPS = {'inView', 'notInView'} +MAX_VIEW_ID = 64 +#: The single-cohort ops this leaf shipped with, kept as PERMANENT aliases and REWRITTEN here: +#: `is part of [one]` is `is any of [that one]`, so a saved view keeps answering and upgrades the +#: next time it is written. Mirrors types.ts COHORT_OP_ALIASES. +COHORT_OP_ALIASES = {'eq': 'anyOf', 'neq': 'noneOf'} +#: How many cohorts one condition may name. Mirrors types.ts MAX_COHORT_IDS. +MAX_COHORT_IDS = 20 + + +def parse_cohort_ids(value): + """The cohorts a leaf names, parsed out of `value`. Mirrors types.ts `cohortIds()`. + + Comma-separated in one string because `FilterRule.value` is what all four layers persist and + round-trip, and a one-element list is byte-identical to what the single-cohort leaf already + stored — so every shipped view parses with no migration. Safe because a cohort id is built + from `[a-z0-9_]` only (modules/cohort.new_id), so a comma cannot occur inside one. + """ + out = [] + for raw in ('' if value is None else str(value)).split(','): + cid = raw.strip()[:120] + if not cid or cid in out: + continue + out.append(cid) + if len(out) >= MAX_COHORT_IDS: + break + return out +# Airtable allows 3 nesting levels (root conditions -> group -> group), then grays +# the button out. MAX_FILTER_DEPTH in types.ts must stay in lock-step with this. +MAX_FILTER_DEPTH = 3 +MAX_FILTER_NODES = 100 # total nodes across the whole tree +MAX_FILTER_SIBLINGS = 50 # per level + + +#: Shape of a measure condition's date window. aios_grid has ZERO app-internal imports by +#: design, so it does NOT know the window VOCABULARY — `harness/windows.py` owns that, mirrored +#: in `customer-grid/windows.ts`, and a third copy here is exactly the drift those two already +#: need a gate to prevent. This validates SHAPE only. +WINDOW_MAX_N = 3650 + + +def _clean_window(raw): + """Structural passthrough for a measure condition's `{kind, n?, from?, to?}` window. + + Meaning is NOT checked here: an unrecognised `kind` survives this function and is REFUSED by + `harness.measure_filter.resolve_rule`, the layer that owns the vocabulary. Splitting it this + way keeps the grid module reusable and keeps one definition of what "last quarter" means. + """ + if not isinstance(raw, dict): + return None + kind = raw.get('kind') + if not isinstance(kind, str) or not kind or len(kind) > 40: + return None + out = {'kind': kind} + if raw.get('n') is not None: + try: + out['n'] = max(1, min(int(raw['n']), WINDOW_MAX_N)) + except (TypeError, ValueError): + return None + for side in ('from', 'to'): + if raw.get(side) not in (None, ''): + out[side] = str(raw[side])[:32] + return out + + +def _clean_rhs(raw, valid_keys): + """CG-9 — validate `{kind, colId, window?}`, the "compare against another attribute" side. + + SHAPE and KEY only: `colId` must be something this table has (the caller widens `valid_keys` + with the measure keys, exactly as it does for the left side), and a measure rhs must carry a + window. What the window MEANS is `harness/windows.py`'s business, same split as `_clean_window`. + """ + if not isinstance(raw, dict): + return None + kind = raw.get('kind') + if kind not in ('field', 'measure', 'stat'): + return None + if kind == 'stat': + # A STATISTIC carries no column: the population is the comparand. Shape only — which + # statistics exist is `harness/measure_filter.STATS`'s business, and an unrecognised one + # is REFUSED there rather than guessed at, exactly like an unrecognised window kind. + stat = raw.get('stat') + if not isinstance(stat, str) or not stat or len(stat) > 24: + return None + return {'kind': 'stat', 'stat': stat} + col = raw.get('colId') + if col not in valid_keys: + return None + out = {'kind': kind, 'colId': col} + if kind == 'measure': + window = _clean_window(raw.get('window')) + if window is None: + return None # a measure comparand with no period is not a question + out['window'] = window + return out + + +#: View DISPLAY MODES beside the grid (wave-6 item 10; 'map' wave-7 W11; 'dashboard' wave-8 +#: I19). Mirrors customer-grid/types.ts DISPLAY_MODES; 'grid' is what an absent/unknown +#: display means, so it is never stored. +#: ⚠ 'dashboard' is RETAINED FOREVER (wave-9 I10, contract C2). The owner renamed the mode to +#: "Chart" (a Dashboard MODULE is coming and the two would collide), but this set is the +#: gatekeeper for a STORED value: `_clean_display` DROPS an unknown mode, so removing +#: 'dashboard' here would silently downgrade every already-saved chart view to grid — and live +#: views are sitting in mode:'dashboard' right now (wave 8's own close-out records one). The +#: rename is therefore a stored-value MIGRATION, not a constant rename: accept 'dashboard' on +#: READ forever, only ever WRITE 'chart'. +#: ⭐ WAVE-27 item 8 (owner ruling R2, contract C3): 'swipe' — a DECK of the records whose bound +#: single-select is EMPTY, triaged one at a time by swiping left or right into two of that +#: field's options. Landed here FIRST and in the same change as the client registry, which is +#: the whole reason the two modes above it needed a staged hold: `_clean_display` DROPS an +#: unknown mode, so a client that offers a mode this set does not carry lets a user build a view +#: that silently reverts to a grid on the next read. +#: ⭐⭐ WAVE-29 R6/R7 (owner item 10, contract C3) — 'form', which CLOSES D-90. The client has +#: carried `form` in its union, with an icon, a label and a tone, since wave 23; this set never +#: did, so `_clean_display` DROPPED both the mode and the `display.form` spec on every write — +#: while `routes_forms.py` reads exactly that key to serve the public submit door. The public door +#: has therefore been live and UNREACHABLE for two waves: not broken, just impossible to point at +#: anything. The mirror is one name, and it is the half nobody could see was missing because +#: BOTH sides were individually consistent. +#: ⚠ Being a legal stored mode is NOT the same as being offered: `form` is deliberately held out +#: of the client's `CREATABLE_MODES` until `CustomerGrid` mounts a renderer for it (the hold law +#: written into `iconShapes.ts`, and now machine-enforced in BOTH directions by +#: `verify_icons.py::mode_parity` — offering an unmounted mode is red, and mounting an unoffered +#: one is red too, so the hold cannot outlive its reason the way wave 27's did). +#: ⭐⭐ WAVE-37 T20 (owner item 10, ruling R2, contract C2) — 'script', and the reason it lands +#: here is NOT the one the PRD gives. The PRD says the Custom View is invisible because this set +#: omits the name so `_clean_display` drops the mode on every write. MEASURED, that write never +#: happens: a script view is a rail PROJECTION built client-side by +#: `CustomerGrid.tsx::scriptProjectionView`, its panel is gated on `activeScriptId !== null` +#: (a row in E's per-database store), there is no `displayMode === "script"` branch at all, and the +#: autosave effect reads `views.find(...)` while the projections are appended to `railViews` ALONE. +#: Nothing was ever dropped, because nothing was ever sent. +#: ⭐ WHAT THE NAME ACTUALLY BUYS is the PRECONDITION for the create door the owner asked for: a +#: script view can only be offered through the display-mode picker once a view STORED as +#: `mode: 'script'` survives a round trip, and until this line it could not. The picker entry +#: itself is W37-T41 (lane E) plus `CREATABLE_MODES`, and `_test/icons.test.ts::HELD_MODES` keeps +#: the hold with its release condition rewritten to say so. +#: ⚠ CONTRACT C2 MAKES THIS SET AND `customer-grid/types.ts::DISPLAY_MODES` IDENTICAL NAME FOR +#: NAME, and `verify_fields_contract.py` now asserts it in BOTH directions. That retires the staged +#: hold every mode from `timeseries` to `form` was landed with — a client-first name is now RED +#: rather than merely risky. The protection the hold gave is structural instead: both halves are in +#: one lane's fence, so they ship in one change, and whether a mode may be CREATED is a separate +#: question answered by `CREATABLE_MODES` / `HELD_MODES`. +DISPLAY_MODES = {'grid', 'list', 'calendar', 'kanban', 'map', 'dashboard', 'chart', + 'timeseries', 'catalog', 'swipe', 'form', 'script'} + +#: W37-T20 (C2) — the cap on `display.script.id`. Script-view ids are minted server-side as +#: `'sv_' + secrets.token_urlsafe(9)` (`routes_script_views.py::create_script_view`), so ~15 chars; +#: 64 is a bound on a hostile payload, not an opinion about the format. +MAX_SCRIPT_ID = 64 + +#: ⭐⭐ WAVE-29 R7 (owner item 10, contracts C3/C4) — THE FORM INTERFACE's stored spec, at +#: `views[].config.display.form`. The public door (`aios-web/api/routes_forms.py`) has read +#: exactly this key since wave 23 and NOTHING HAS EVER BEEN ABLE TO WRITE IT: `form` was not a +#: legal mode and this function had no branch for the key, so every spec a client sent was dropped +#: on the way in. That is D-90 stated precisely — not a broken feature, an unreachable one. +#: +#: ⛔ THE TOKEN IS NOT HERE, AND IT NEVER WILL BE. A share token that rides the wire is a token a +#: browser can CHOOSE, and `routes_forms._resolve` walks tenants and answers with the FIRST match — +#: so one tenant setting its token to another tenant's value would silently receive that tenant's +#: submissions. The token is minted server-side and lives in a bucket no client write can reach; +#: `_clean_form` drops any `token` key that arrives here, rather than validating its shape. +#: ⭐⭐ WAVE-37 T21 (owner items 1/12, rulings R5+R6, contract C3) — THE PER-VIEW CATALOG SPEC, +#: stored at `views[].config.display.catalog`. +#: +#: ⛔⛔ IT IS **`catalog`**, SINGULAR, AND IT IS NOT `catalogs`. Both keys live on `display`, they +#: are one letter apart, and that is the single most likely mistake anybody touching this file will +#: make. They are different things and BOTH are live: +#: · `catalogs` (wave-18 C6) — a LIST of up to 12 print artifacts, each with its own paper, +#: brand and pages of kind cover/intro/section/gallery. Authored CONTENT. UNTOUCHED by T21. +#: · `catalog` (wave-37 C3) — THIS view's catalog SETTINGS: which column is the picture, which +#: is the title (R5: per view, so two catalog views on one database may differ), plus the page +#: order and the free-placed items R6 asks for. +#: Neither replaces the other, and neither validator reads the other's key. +#: +#: ⛔ THE MIGRATION IS A RULE ABOUT WHAT THIS FUNCTION MUST **NOT** DO (R6, C3). An item authored +#: before free placement has no `x`/`y`, and an ABSENT coordinate must read back ABSENT — never +#: normalised to 0. Defaulting to zero would pile every pre-existing item at the origin, which is +#: "a catalog that silently loses its layout is a FAIL, not a migration" stated as code. The +#: RENDERER turns an absent coordinate into flow position N; the host's whole job is to not destroy +#: the distinction it reads. +#: ⚠ `w`/`h` carry the OPPOSITE asymmetry, deliberately: a zero WIDTH is an invisible item, so 0 +#: is refused and absent means "the renderer's default size". A zero X is the top-left corner and is +#: a real, storable position. Two keys of one shape with two rules — the `kanbanClamp` family — +#: which is why each is written out rather than looped. +MAX_CATALOG_SECTIONS = 20 +#: A bound on a hostile payload, not an opinion about the renderer's coordinate space: D owns what +#: the numbers MEAN. Stored to 2 decimals, finer than any print surface resolves. +MAX_CATALOG_COORD = 10000 +#: The page `order` bound. ABSENT => that page's ARRAY INDEX, and `order` WINS where present; ties +#: break by array index. Decided HERE rather than left to the renderer, because two readers guessing +#: differently about page order is exactly the divergence contract C3 exists to close. +MAX_CATALOG_ORDER = 9999 + + +def _clean_catalog_coord(raw, allow_zero): + """One free-placement number, or None. `allow_zero` False is the w/h leg (see the header). + + ⛔ THE ROUNDING IS `floor(v * 100 + 0.5) / 100` AND NOT `round(v, 2)`, AND THAT IS THE WHOLE + POINT OF THIS FUNCTION EXISTING SEPARATELY. Python's `round` is BANKER'S rounding and + JavaScript's `Math.round` is half-up-toward-+Infinity, so the two engines disagree on every + exact half: `round(0.125, 2)` is 0.12 here and 0.13 in the browser. One dragged item landing two + hundredths apart on the two engines is a `display` object that never compares equal — the + silent mirror drift contract C2 exists to prevent, arriving through arithmetic instead of + through a missing key. The expression below is `Math.floor(v * 100 + 0.5) / 100` written in + Python, evaluated in the same IEEE doubles, so the two agree bit for bit. + ⚠ AND AN INTEGRAL RESULT IS EMITTED AS AN `int`: `12.0` serialises as `12.0` in Python and + `12` in JSON.stringify, which is one stored value with two spellings. + """ + if isinstance(raw, bool) or not isinstance(raw, (int, float)): + return None + if raw != raw or raw in (float("inf"), float("-inf")): # NaN / +-inf + return None + val = math.floor(float(raw) * 100 + 0.5) / 100 + if val > MAX_CATALOG_COORD or val < -MAX_CATALOG_COORD: + return None + if not allow_zero and val <= 0: + return None + return int(val) if val == int(val) else val + + +def _clean_catalog_item(raw): + """One free-placed item: `{code, x?, y?, w?, h?}`. `code` REQUIRED; every coordinate optional. + + Key EMISSION ORDER is fixed and mirrored byte-for-byte by `types.ts::cleanCatalogSpec` — an + accepted save has to read back identically on both engines, and this pair is where that is + decided. + """ + if not isinstance(raw, dict): + return None + code = raw.get("code") + if not isinstance(code, str) or not code.strip(): + return None + out = {"code": code.strip()[:CATALOG_CODE_MAX]} + for key, allow_zero in (("x", True), ("y", True), ("w", False), ("h", False)): + val = _clean_catalog_coord(raw.get(key), allow_zero) + if val is not None: + out[key] = val + return out + + +def _clean_catalog_spec(raw, valid_keys): + """C3 — `display.catalog`, fail-closed. See the header block above for what it is NOT. + + Field refs follow the `dateField`/`stackField` rule this module already runs: a ref naming a + deleted column is dropped INDIVIDUALLY and the renderer falls back to its per-mode default. It + never costs the user their pages — the `charts` per-entry-drop precedent, applied to the one + payload here that is AUTHORED rather than derived from the rows. + """ + if not isinstance(raw, dict): + return None + out = {} + for ref in ("imageField", "titleField"): + if raw.get(ref) in valid_keys: + out[ref] = raw[ref] + pages_raw = raw.get("pages") + pages = [] + if isinstance(pages_raw, list): + # ONE cumulative item budget across the whole spec, spent in page order then section order + # — the `_clean_catalogs` construction, so a 40-page catalog cannot smuggle 20,000 items + # past a per-page cap. + budget = MAX_CATALOG_CODES + seen_pages = set() + for raw_page in pages_raw: + if len(pages) >= MAX_CATALOG_PAGES: + break + if not isinstance(raw_page, dict): + continue + pid = str(raw_page.get("id") or "").strip()[:CATALOG_ID_MAX] + if not pid or pid in seen_pages: + continue + seen_pages.add(pid) + page = {"id": pid} + order = raw_page.get("order") + if (isinstance(order, int) and not isinstance(order, bool) + and 0 <= order <= MAX_CATALOG_ORDER): + page["order"] = order + sections_raw = raw_page.get("sections") + sections = [] + if isinstance(sections_raw, list): + seen_sections = set() + for raw_section in sections_raw: + if len(sections) >= MAX_CATALOG_SECTIONS: + break + if not isinstance(raw_section, dict): + continue + sid = str(raw_section.get("id") or "").strip()[:CATALOG_ID_MAX] + if not sid or sid in seen_sections: + continue + seen_sections.add(sid) + section = {"id": sid} + items_raw = raw_section.get("items") + items = [] + if isinstance(items_raw, list) and budget > 0: + for raw_item in items_raw: + item = _clean_catalog_item(raw_item) + if item is None: + continue + items.append(item) + budget -= 1 + if budget <= 0: + break + # ⚠ NOT de-duplicated by `code`. The same product legitimately appears twice in + # one section at two positions — that is what free placement IS — and wave-18's + # "deduped WITHIN a page" rule was written for a LISTING, where a repeat is a + # mistake. Here it is the feature. + if items: + section["items"] = items + sections.append(section) + # An EMPTY section is KEPT: a person adds a New Section and then drags into it, and + # dropping it would delete the thing they just made, between two autosaves. + if sections: + page["sections"] = sections + pages.append(page) + if pages: + out["pages"] = pages + # A spec that reduces to nothing is DROPPED, never stored as `{}`. "No image field, no title + # field, no pages" IS the unconfigured state, and `{}` would be its second spelling — the + # no-churn law every literal-only key in `_clean_display` follows. + return out or None + + +FORM_ACCESS = ('public', 'emails') +MAX_FORM_FIELDS = 60 +MAX_FORM_EMAILS = 200 +MAX_FORM_TITLE, MAX_FORM_DESC, MAX_FORM_SUBMIT = 120, 1000, 60 +MAX_FORM_EMAIL = 254 + +#: The field types a form may COLLECT, as an ALLOW-LIST rather than a list of exclusions — the +#: fail-closed direction, because the cost of the two mistakes is not symmetric. A type missing +#: from here is a question the builder cannot ask yet; a type wrongly present is a public door +#: writing values the column cannot mean (an `image` with no upload channel, a `json` document +#: typed into a text box, a `link` naming a record id a stranger guessed). +#: ⚠ NOT sufficient on its own, and the reason is a shape this codebase has been bitten by before: +#: a METRIC bag rides ANY field type (`core/user_tables.py` — `metric` is not a field kind), so an +#: `int` column can be machine-computed while passing this list. `routes_forms` therefore asks +#: `user_tables.is_computed_cell` as well — one evaluator for "is this computed", reused rather +#: than re-derived ([[one-evaluator-per-question]]). +#: Client mirror: `customer-grid/FormInterface.tsx` FORM_FIELD_TYPES; `verify_forms.py` compares +#: the two files name-for-name. +FORM_FIELD_TYPES = ('text', 'select', 'multiselect', 'int', 'currency', 'pct', 'date', + 'checkbox', 'phone', 'email', 'url', 'rating') + +#: Deliberately looser than a full RFC parse and stricter than `routes_forms._clean_values`' "@ in +#: it": this list decides who MAY SUBMIT, so a typo that silently locks a colleague out is the +#: expensive failure, not an odd address that gets in. +_FORM_EMAIL = re.compile(r'^[^@\s,;]+@[^@\s,;]+\.[^@\s,;]+$') + + +def _clean_form(raw, valid_keys): + """One form spec, fail-closed. Returns None when nothing is configured. + + ⚠ FIELD ORDER IS THE FORM'S OWN and is preserved here, not re-derived from the schema: the + builder let somebody arrange these questions, and sorting them by column order would silently + rearrange a live form every time a column was added (`_public_form` states the same rule from + the serving end). + + ⚠ PARTIAL-DROP, not whole-key drop, and the asymmetry against `swipe` above is deliberate. A + swipe binding is one three-part machine: two of its parts is not a degraded deck, it is a deck + that can never write. A form is a LIST of questions — losing the column behind question three + costs the asker question three, and taking the whole form away because one field was deleted + would be a far larger loss than the one that happened. + """ + if not isinstance(raw, dict): + return None + fields, seen = [], set() + for k in (raw.get('fields') or [])[:MAX_FORM_FIELDS]: + if k in valid_keys and k not in seen: + seen.add(k) + fields.append(k) + out = {} + if fields: + out['fields'] = fields + # A required flag on a question the form no longer asks is not a rule, it is a trap: the + # submitter can never satisfy it and the sentence names a field they cannot see. + req = [k for k in dict.fromkeys(raw.get('required') or []) if k in seen] + if req: + out['required'] = req + for key, cap in (('title', MAX_FORM_TITLE), ('desc', MAX_FORM_DESC), + ('submitLabel', MAX_FORM_SUBMIT)): + text = str(raw.get(key) or '').strip()[:cap] + if text: + out[key] = text + # `public` is the ABSENT default (the `kanbanClamp` law: one way to say one thing), so only + # the restrictive value is ever stored. ⇒ a spec that loses its `access` key fails OPEN, which + # is why `emails` is what gets written rather than a `public: false`. + if raw.get('access') == 'emails': + out['access'] = 'emails' + emails = [] + for e in (raw.get('emails') or [])[:MAX_FORM_EMAILS]: + e = str(e or '').strip().lower()[:MAX_FORM_EMAIL] + if _FORM_EMAIL.match(e) and e not in emails: + emails.append(e) + # Kept even while `access` is public: a person toggling the door open to test it and back + # again must not lose the list of people they typed. It is never served publicly. + if emails: + out['emails'] = emails + return out or None + + +#: C3 — the swipe binding's option cap. `leftOption`/`rightOption` are stored VALUES of a +#: single-select, and `_clean_options` trims every choice to 120 chars, so this is that same +#: number rather than a second opinion about it: a longer string cannot name a real option, and +#: a SHORTER cap here would silently refuse a binding to a legal one. +MAX_SWIPE_OPTION = 120 + +#: C-DISP (wave 2026-08-02): the time-series view's bucket vocabulary and caps, plus the +#: calendar-summary metric cap. types.ts mirrors these as TS_BUCKETS / TS_MAX_LAST_N / +#: TS_MAX_FIELDS / MAX_CALENDAR_METRICS, and cleanDisplay applies the same per-entry drops, +#: so an accepted save reads back byte-identically on both engines. +TS_BUCKETS = {'week', 'month', 'quarter', 'year'} +TS_MAX_LAST_N = 120 +TS_MAX_FIELDS = 12 +MAX_CALENDAR_METRICS = 4 +_ISO_DAY = re.compile(r'^\d{4}-\d{2}-\d{2}$') + +#: C6-CATALOG (wave 18) — the catalog view's vocabulary and caps. types.ts mirrors every name +#: below, and `cleanDisplay` applies the same drops in the SAME ORDER, so an accepted save reads +#: back identically on both engines. The code budget is the order-sensitive one — see +#: `_clean_catalogs`. +MAX_CATALOGS = 12 +MAX_CATALOG_PAGES = 40 +MAX_CATALOG_CODES = 500 # cumulative across ONE catalog's pages, spent in PAGE ORDER +CATALOG_PAPERS = {'letter', 'a4', 'tabloid'} +CATALOG_ORIENTATIONS = {'portrait', 'landscape'} +CATALOG_QUALITIES = {'web', 'print'} +CATALOG_PAGE_KINDS = {'cover', 'intro', 'section', 'gallery'} +CATALOG_COLS = (2, 3, 4) +CATALOG_ID_MAX, CATALOG_NAME_MAX = 40, 80 +CATALOG_TITLE_MAX, CATALOG_BODY_MAX, CATALOG_CODE_MAX = 120, 2000, 60 +_HEX6 = re.compile(r'^#[0-9A-Fa-f]{6}$') + +#: Wave 14 C-ACC ([[loopable-wave14-split]]; rulings R2/R3). Mirrored by types.ts +#: TS_DELTA_KINDS / TS_MAX_CUSTOM_ROWS / TS_MAX_STYLES — the C-DISP byte-identical law. +TS_DELTA_KINDS = ('abs', 'pct', 'yoy', 'ytd') +TS_MAX_CUSTOM_ROWS = 12 +TS_MAX_STYLES = 200 +#: R2 — a formula row's `expr` is stored VERBATIM and NEVER parsed here (evaluation is client +#: law; the client refuses unknown refs/cycles/div-zero itself). The charset wall is the whole +#: server-side contract: row refs `[...]`, arithmetic, numbers — no markup, no control chars. +_TS_EXPR_OK = re.compile(r'^[A-Za-z0-9_ .+\-*/()\[\]]+$') + +#: Old wire value -> the value we store today. Applied AFTER the membership test so an unknown +#: mode is still rejected rather than accidentally aliased. +_LEGACY_MODES = {'dashboard': 'chart'} + +#: Chart kinds a chart-mode view may hold (wave-8 I19, contract C2). Mirrors the client's +#: union. Deliberately small: the owner asked to "start with simple charts" and expand, and a +#: kind the client cannot draw is worse than one that does not exist yet. +#: Wave-16 C-CHARTCAP: + 'table' — the group-by aggregate table (by-rep / by-BU / +#: top-customers, the third Sales block shape). Client renderer: DashboardView's +#: GroupTableView over salesParity.tableFromSpec. +CHART_KINDS = {'bar', 'line', 'area', 'donut', 'kpi', 'table'} +CHART_AGGS = {'sum', 'avg', 'count', 'min', 'max'} + +#: ⭐ WAVE-29 C7 (item 17) — THE COLUMN-SUMMARY vocabulary: what a FIELD's `agg` may be, which is +#: what the grid's totals row and its per-group subtotals compute. ORDERED, because the order is +#: the picker's order; membership tests read it as a tuple perfectly well. +#: +#: ⛔ IT IS NOT `CHART_AGGS` AND THE TWO MUST NOT BE MERGED, however alike they look. `CHART_AGGS` +#: gatekeeps a STORED value with live data behind it (`charts[].agg`, `calendarMetrics[].agg`): +#: `_clean_chart` falls back to 'sum' on an unknown agg and `_clean_display` DROPS a whole +#: calendarMetrics entry, so renaming its 'avg' would silently turn every saved chart into a sum +#: and delete calendar cards, with nothing red. A chart's aggregation and a column's summary are +#: also different questions — one reduces a SERIES, the other a COLUMN — and one list serving both +#: would have to be the intersection of what each can express. +#: +#: ⭐ `average`, NOT `avg`, and the tie is broken by the vocabulary we cannot rename: `ROLLUP_FNS` +#: (`core/user_tables.py`, 16 names, 47 rollups live in production) already spells it `average`, +#: and it is the aggregate vocabulary a user actually reads today. Spelling it `avg` here would +#: give the product two words for one operation on two menus a click apart. +#: +#: ⚠ `median` is net-new: it is in NEITHER `CHART_AGGS` nor `ROLLUP_FNS`, so a Median column +#: summary has no rollup equivalent and this list is NOT a subset of either of its neighbours. +#: +#: ⚠ `count` counts ROWS in the scope (the group, or every matched row) — not non-blank cells. +#: `ROLLUP_FNS` splits that hair three ways (count / counta / countall); a column summary does not, +#: and must not grow a second spelling of it. +#: +#: Client mirror: `customer-grid/iconShapes.ts` FIELD_AGGS — ONE client list, imported by +#: `aggregations.ts` and the field editor rather than re-declared, so the only boundary left to +#: police is this one. `verify_icons.py::agg_parity` reads BOTH FILES and compares them. +FIELD_AGGS = ('sum', 'average', 'median', 'min', 'max', 'count') +MAX_CHARTS = 12 #: per view — a dashboard, not an unbounded render loop +MAX_CHART_TITLE = 60 + +#: Wave-9 I11 (contract C2) — chart customisation, host-validated. +#: +#: `palette` names a colour JOB, never a colour. A browser must not be able to post a raw hex: +#: the four names below map to the four jobs a palette can do (identity / magnitude / polarity) +#: and resolve to brand ramps client-side, so a tenant restyle cannot be defeated by a stored +#: literal. STATUS colours (good/warning/serious/critical) are deliberately NOT selectable — +#: they are reserved signal, and reusing them as "series 4" is how a chart starts lying. +CHART_PALETTES = {'brand', 'categorical', 'sequential', 'diverging'} +CHART_FORMATS = {'auto', 'number', 'currency', 'percent', 'compact'} +MAX_AXIS_LABEL = 40 +#: `size` is the I10 drag. Width is in GRID COLUMNS (a 12-column board), height in px. +CHART_W_RANGE = (1, 12) +CHART_H_RANGE = (120, 800) + + +def _clean_chart(raw, valid_keys): + """One dashboard chart, fail-closed. Returns None if the chart cannot be drawn. + + A chart's `y` is optional (absent = count of rows, which is what "how many customers per + state" means). `x` is NOT: a chart with no category axis has nothing to plot against, and + silently keeping it would put an empty card on the dashboard with no way to tell why. + """ + if not isinstance(raw, dict): + return None + kind = raw.get('kind') + if kind not in CHART_KINDS: + return None + x = raw.get('x') + if x not in valid_keys: + return None # dead category ref -> the chart goes, not the board + cid = raw.get('id') + if not isinstance(cid, str) or not cid.strip(): + return None # the client owns chart ids; an unidentified card + # cannot be edited or removed, so it must not persist + out = {'id': cid.strip()[:64], 'kind': kind, 'x': x, + 'agg': raw.get('agg') if raw.get('agg') in CHART_AGGS else 'sum'} + if raw.get('y') in valid_keys: + out['y'] = raw['y'] + else: + # no measurable column -> the only honest aggregation left is "how many rows" + out['agg'] = 'count' + title = raw.get('title') + if isinstance(title, str) and title.strip(): + out['title'] = title.strip()[:MAX_CHART_TITLE] + + # ── wave-9 I11 (contract C2): customisation ──────────────────────────────────────────── + # `splitBy` is the field whose values become the SERIES. It is deliberately not called + # `colorBy`: that name already means two other things here (`config.colorBy` = row + # colouring, `display.colorField` = map pin colour) and a third sense would be unreadable. + if raw.get('splitBy') in valid_keys and raw['splitBy'] != x: + out['splitBy'] = raw['splitBy'] + # Stacking is only a question once there are series to stack, and only for the two kinds + # that can express it. Anywhere else it is dropped rather than stored as a lie the client + # would have to re-decide. + if out.get('splitBy') and kind in ('bar', 'area') and raw.get('stacked') is True: + out['stacked'] = True + if raw.get('palette') in CHART_PALETTES: + out['palette'] = raw['palette'] + + axis = raw.get('axis') + if isinstance(axis, dict): + # ⛔ ONE y-scale, always. There is no second-axis key here and there must never be: + # two y-scales on one frame can manufacture any correlation you like by rescaling, and + # the honest alternatives are two charts, small multiples, or indexing to a common base. + # Ruled explicitly in contract C2 against the "Tableau versatility" brief. + clean_axis = {} + for side in ('x', 'y'): + spec = axis.get(side) + if not isinstance(spec, dict): + continue + one = {} + lab = spec.get('label') + if isinstance(lab, str) and lab.strip(): + one['label'] = lab.strip()[:MAX_AXIS_LABEL] + if spec.get('format') in CHART_FORMATS: + one['format'] = spec['format'] + if one: + clean_axis[side] = one + if clean_axis: + out['axis'] = clean_axis + + size = raw.get('size') + if isinstance(size, dict): + one = {} + for key, (lo, hi) in (('w', CHART_W_RANGE), ('h', CHART_H_RANGE)): + try: + one[key] = max(lo, min(hi, int(size[key]))) + except (KeyError, TypeError, ValueError): + pass # a partial size is fine: the client defaults the missing axis + if one: + out['size'] = one + + # ── Wave 14 R3 ([[loopable-wave14-split]]): a METRIC chart may carry a PERIOD — the + # trend-over-buckets encoding. Kept only when the chart's value field is measure-backed: + # a category column has no time dimension, and a stored period on it would promise a + # trend the TS channel must refuse. `span` is meaningful only beside `bucket`. + if isinstance(out.get('y'), str) and out['y'].startswith('measure_'): + if raw.get('bucket') in TS_BUCKETS: + out['bucket'] = raw['bucket'] + sp = raw.get('span') + if isinstance(sp, dict): + n = sp.get('lastN') + if (isinstance(n, int) and not isinstance(n, bool) + and 1 <= n <= TS_MAX_LAST_N): + out['span'] = {'lastN': n} + # ── Wave-16 C-CHARTCAP: the YoY companion. Kept ONLY where it can mean something — + # beside a kept bucket (the compare series) or on a sum-of-metric KPI (the delta + # line). Anything else is a stored claim the renderer would have to re-refuse. + # Mirrors the client's cleanCharts rule key for key. + if raw.get('compare') == 'prior_year' and ( + out.get('bucket') or (kind == 'kpi' and out.get('agg') == 'sum')): + out['compare'] = 'prior_year' + return out + + +def _clean_catalog_page(raw, budget): + """C6-CATALOG — one page of a catalog. Returns `(page | None, codes_spent)`. + + `budget` is what is LEFT of the catalog's 500-code allowance. Codes are deduped WITHIN a + page and not across the catalog: a product legitimately appears on a gallery page and again + in its section listing, and de-duplicating globally would silently delete the second + appearance. The budget is spent in page order, so a catalog that runs out loses the TAIL of + its last pages — never a random scatter, and never a page (a page with no products is a + heading the user can still see and fix). + """ + if not isinstance(raw, dict): + return None, 0 + page_id = str(raw.get('id') or '')[:CATALOG_ID_MAX] + kind = raw.get('kind') + if not page_id or kind not in CATALOG_PAGE_KINDS: + return None, 0 + out = {'id': page_id, 'kind': kind} + for key, cap in (('title', CATALOG_TITLE_MAX), ('body', CATALOG_BODY_MAX), + ('imageCode', CATALOG_CODE_MAX)): + v = raw.get(key) + if isinstance(v, str) and v: + out[key] = v[:cap] + products = raw.get('products') + if isinstance(products, list) and budget > 0: + clean_p, seen_p = [], set() + for c in products: + if not isinstance(c, str) or not c: + continue + c = c[:CATALOG_CODE_MAX] + if c in seen_p: + continue + seen_p.add(c) + clean_p.append(c) + if len(clean_p) >= budget: + break + if clean_p: + out['products'] = clean_p + layout = raw.get('layout') + if isinstance(layout, dict): + clean_l = {} + cols = layout.get('cols') + if isinstance(cols, int) and not isinstance(cols, bool) and cols in CATALOG_COLS: + clean_l['cols'] = cols + # The kanbanClamp/tsSparkline asymmetry, one per direction: pack and colour SHOW by + # default (the 2027 catalogue shows both), price does NOT (it shows no prices at all). + # So only the opt-OUT is storable for the first two and only the opt-IN for the third — + # a second spelling of a default is how a round trip starts churning. + if layout.get('showPack') is False: + clean_l['showPack'] = False + if layout.get('showColor') is False: + clean_l['showColor'] = False + if layout.get('showPrice') is True: + clean_l['showPrice'] = True + if clean_l: + out['layout'] = clean_l + return out, len(out.get('products') or ()) + + +def _clean_catalogs(raw, valid_keys): + """C6-CATALOG (wave 18) — `display.catalogs`, fail-closed. Returns a list or None. + + A catalog is a PRINT artifact, so the two structural keys that decide how it paginates + (`paper`, `orientation`) are NORMALISED WITH A DEFAULT rather than dropped: a page box with + no size is not a smaller catalog, it is an unrenderable one. Everything else follows the + house rules — unknown keys dropped, per-entry drops never cost the neighbours, empty + sub-objects omitted entirely (`brand`, `fields`, `layout`, `products`) so an absent key and + an empty one are not two spellings of the same nothing. + + `fields` binds the listing lines to real columns (the 2027 listing prints description / SKU / + pack / colour, and `product_data` carries no pack or colour of its own — the user binds + custom fields). Refs are checked against `valid_keys` HERE and not on the client, the same + split `dateField`/`stackField` already run. + """ + if not isinstance(raw, list): + return None + out = [] + for c in raw: + if len(out) >= MAX_CATALOGS: + break + if not isinstance(c, dict): + continue + cat_id = str(c.get('id') or '')[:CATALOG_ID_MAX] + name = c.get('name') + # An EMPTY name is legal (the user cleared the box and will type again) — an ABSENT one + # is a malformed record. The `tsRows` label rule, same reasoning. + if not cat_id or not isinstance(name, str): + continue + cat = {'id': cat_id, 'name': name[:CATALOG_NAME_MAX]} + cat['paper'] = c['paper'] if c.get('paper') in CATALOG_PAPERS else 'letter' + cat['orientation'] = (c['orientation'] + if c.get('orientation') in CATALOG_ORIENTATIONS else 'portrait') + if c.get('quality') in CATALOG_QUALITIES: + cat['quality'] = c['quality'] + brand = c.get('brand') + if isinstance(brand, dict): + clean_b = {} + for k in ('primary', 'accent'): + v = brand.get(k) + if isinstance(v, str) and _HEX6.match(v): + clean_b[k] = v + company = brand.get('company') + if isinstance(company, str) and company: + clean_b['company'] = company[:CATALOG_NAME_MAX] + # An asset CODE (resolved through C2-ASSET), never a URL: an arbitrary host inside + # print CSS is exactly the tokens-not-values rule this contract carries. + logo = brand.get('logo') + if isinstance(logo, str) and logo: + clean_b['logo'] = logo[:CATALOG_CODE_MAX] + if clean_b: + cat['brand'] = clean_b + binds = c.get('fields') + if isinstance(binds, dict): + clean_bind = {k: binds[k] for k in ('name', 'pack', 'color', 'price') + if binds.get(k) in valid_keys} + if clean_bind: + cat['fields'] = clean_bind + pages, budget = [], MAX_CATALOG_CODES + raw_pages = c.get('pages') + if isinstance(raw_pages, list): + for p in raw_pages: + if len(pages) >= MAX_CATALOG_PAGES: + break + page, spent = _clean_catalog_page(p, budget) + if page is None: + continue + budget -= spent + pages.append(page) + # ALWAYS emitted, even empty: a catalog with no pages yet is the state every catalog + # starts in, and dropping the key would make "new" and "corrupt" the same wire value. + cat['pages'] = pages + out.append(cat) + return out or None + + +def _clean_display(raw, valid_keys): + """Structural passthrough for a view's `config.display` (wave-6 item 10), fail-closed. + + `{mode, dateField?, stackField?, titleField?, colorField?, sizeField?, charts?}` — mode + must be a known non-grid mode (grid is the absent default, so storing it would be a second + way to say nothing); every field ref must name a field this table has (a ref to a deleted + field is DROPPED and the client falls back to its per-mode default); unknown keys are + dropped. What each mode MEANS — calendar wants a date-family field, kanban a select-family + stack, map a single-select to colour by and a numeric to size by — is the client's + business: it is the only layer that renders them, and a wrong-typed ref degrades to that + surface's default rather than to an error (the `_clean_window` split). + + Wave-8 (contract C2) adds the map encodings (`colorField` I3, `sizeField` I5) and + dashboard `charts` (I19). A chart whose x/y names a deleted field is dropped INDIVIDUALLY — + never the whole array, because losing one column should not cost the user a dashboard they + spent time building. + + W33-T45 (contract C1, ruling R5) adds `published` + `publishAccess`. ⚠ THE HALF OF THE ROUND + TRIP THIS FUNCTION CANNOT ENFORCE: `customer-grid/types.ts::cleanDisplay` is a SECOND + normalizer, in the browser, which rebuilds the config key by key on every autosave and drops + anything it does not name. A key accepted here and unknown there dies on the next column + resize — silently, because the client then POSTs the stripped config and the host REPLACES + the stored one. "The host accepts it" is half a round trip; that file is the other half. + """ + if not isinstance(raw, dict): + return None + mode = raw.get('mode') + if mode not in DISPLAY_MODES or mode == 'grid': + return None + # Wave-9 I10 (C2): normalise the legacy wire value AFTER the membership test, so an unknown + # mode is still rejected rather than accidentally aliased into a real one. Every already + # saved 'dashboard' view reads back as 'chart' from here on; nothing writes 'dashboard'. + mode = _LEGACY_MODES.get(mode, mode) + out = {'mode': mode} + # ⭐ W37-T28 (owner item 15) — `coordField`: ONE column holding `"lat,lon"`, so a map is no + # longer a customers-only feature. It joins this loop rather than getting a clause of its own + # because it IS a field ref with the same drop-if-deleted rule; `customer-grid/types.ts` + # gains it in the SAME change, which is contract C2's whole point. + for ref in ('dateField', 'stackField', 'titleField', 'colorField', 'sizeField', 'coordField'): + if raw.get(ref) in valid_keys: + out[ref] = raw[ref] + # ── C-DISP (wave 2026-08-02) ───────────────────────────────────────────────────────── + # kanbanClamp: stored ONLY as the literal opt-OUT. Absent means clamped — the new + # standardized default — so storing True would be a second way to say nothing (the same + # rule that keeps mode:'grid' out of the store). + if raw.get('kanbanClamp') is False: + out['kanbanClamp'] = False + if raw.get('calendarMode') in ('records', 'summary'): + out['calendarMode'] = raw['calendarMode'] + metrics = raw.get('calendarMetrics') + if isinstance(metrics, list): + clean_m, seen_m = [], set() + for m in metrics[:MAX_CALENDAR_METRICS]: + # Dropped INDIVIDUALLY (the charts precedent): one dead metric must not cost the + # user the summary card they configured around it. + if not isinstance(m, dict): + continue + mid = str(m.get('id') or '')[:40] + if (not mid or mid in seen_m or m.get('field') not in valid_keys + or m.get('agg') not in CHART_AGGS): + continue + seen_m.add(mid) + clean_m.append({'id': mid, 'field': m['field'], 'agg': m['agg']}) + if clean_m: + out['calendarMetrics'] = clean_m + if raw.get('tsBucket') in TS_BUCKETS: + out['tsBucket'] = raw['tsBucket'] + span = raw.get('tsSpan') + if isinstance(span, dict): + clean_span = {} + n = span.get('lastN') + if isinstance(n, int) and not isinstance(n, bool) and 1 <= n <= TS_MAX_LAST_N: + clean_span['lastN'] = n + else: + f, t = span.get('from'), span.get('to') + f = f if isinstance(f, str) and _ISO_DAY.match(f) else None + t = t if isinstance(t, str) and _ISO_DAY.match(t) else None + if f and t and f > t: + f, t = t, f + if f: + clean_span['from'] = f + if t: + clean_span['to'] = t + if clean_span: + out['tsSpan'] = clean_span + ts_fields = raw.get('tsFields') + if isinstance(ts_fields, list): + clean_f, seen_f = [], set() + for k in ts_fields[:TS_MAX_FIELDS]: + if k in valid_keys and k not in seen_f: + seen_f.add(k) + clean_f.append(k) + if clean_f: + out['tsFields'] = clean_f + # ── Wave 14 C-ACC ([[loopable-wave14-split]] R2; items 17/18) ──────────────────────── + deltas = raw.get('tsDeltas') + if isinstance(deltas, list): + clean_d, seen_d = [], set() + for d in deltas: + if d in TS_DELTA_KINDS and d not in seen_d: + seen_d.add(d) + clean_d.append(d) + if clean_d: + out['tsDeltas'] = clean_d + # Gridlines: stored ONLY as the literal opt-OUT (absent = shown), sparkline ONLY as the + # literal opt-IN (absent = off) — the kanbanClamp asymmetry, one per direction. + if raw.get('tsGridlines') is False: + out['tsGridlines'] = False + if raw.get('tsSparkline') is True: + out['tsSparkline'] = True + rows = raw.get('tsRows') + if isinstance(rows, list): + clean_r, seen_r = [], set() + for r in rows[:TS_MAX_CUSTOM_ROWS]: + if not isinstance(r, dict): + continue + rid = str(r.get('id') or '')[:40] + r_kind = r.get('kind') + if not rid or rid in seen_r or r_kind not in ('note', 'formula'): + continue + label = r.get('label') + if not isinstance(label, str): + continue # ABSENT label = malformed; an EMPTY one is a legal spacer row + # (GRID's dated asymmetry amendments, 2026-08-02) + one = {'id': rid, 'kind': r_kind, 'label': label.strip()[:120]} + if r_kind == 'formula': + expr = r.get('expr') + if (isinstance(expr, str) and expr.strip() + and len(expr) <= 200 and _TS_EXPR_OK.match(expr)): + one['expr'] = expr + # else: keep the ROW, drop the EXPR — it renders "—". Vanishing the row + # would delete the user's label to punish their arithmetic (GRID's dated + # asymmetry amendment; the calendarMetrics per-entry-drop precedent). + seen_r.add(rid) + clean_r.append(one) + if clean_r: + out['tsRows'] = clean_r + styles = raw.get('tsStyles') + if isinstance(styles, dict): + clean_s = {} + for s_key, s_val in styles.items(): + if len(clean_s) >= TS_MAX_STYLES: + break # capped, not truncated silently: the gate names this + if not isinstance(s_key, str) or not s_key or len(s_key) > 96: + continue # key = rowId or "rowId:colKey" — the client's grammar + if not isinstance(s_val, dict): + continue + one = {} + if s_val.get('bold') is True: + one['bold'] = True + if s_val.get('line') is True: + one['line'] = True + if one: + clean_s[s_key] = one + if clean_s: + out['tsStyles'] = clean_s + charts = raw.get('charts') + if isinstance(charts, list): + clean = [c for c in (_clean_chart(x, valid_keys) for x in charts[:MAX_CHARTS]) if c] + # de-dupe by id: two cards sharing an id are one card as far as the client's keyed + # render is concerned, and the second would silently shadow the first + seen, uniq = set(), [] + for c in clean: + if c['id'] in seen: + continue + seen.add(c['id']) + uniq.append(c) + if uniq: + out['charts'] = uniq + # ── C6-CATALOG (wave 18) ───────────────────────────────────────────────────────────── + catalogs = _clean_catalogs(raw.get('catalogs'), valid_keys) + if catalogs: + out['catalogs'] = catalogs + # ── ⭐⭐ WAVE-37 C3 (T21) — `catalog`, SINGULAR, and it is NOT the line above it. See the + # header on `_clean_catalog_spec` for the two-keys-one-letter-apart warning; this call site is + # where a reader is most likely to "fix" one into the other. + catalog = _clean_catalog_spec(raw.get('catalog'), valid_keys) + if catalog: + out['catalog'] = catalog + # ── ⭐ WAVE-27 C3 (item 8 / R2): the swipe binding ──────────────────────────────────── + # `{fieldKey, leftOption, rightOption}` — WHOLE-KEY drop, never a partial one, and that + # asymmetry against `charts`/`calendarMetrics` above is the point rather than an oversight. + # Those are LISTS of independent cards, so losing one entry costs the user one card. This is + # a single three-part BINDING: a swipe view holding a fieldKey with one option, or two + # options and no field, is not a degraded swipe view — it is a deck that can never write + # anything, rendered as though it were configured. Dropping the key entirely puts the view + # back in its honest unconfigured state, which is the one state the client has a UI for. + # + # ⚠ What this CANNOT check, deliberately, and why the client must: whether `fieldKey` names + # a SELECT, and whether the two options are still in that select's vocabulary. `valid_keys` + # is a key set, and the docstring above draws this exact line — "what each mode MEANS ... is + # the client's business". So SwipeView owns three losses this function is blind to (field + # deleted, field retyped away from select, option removed) and must SHOW each one rather + # than fall back to the first option, per the `viewModes.tsx` house rule. + swipe = raw.get('swipe') + if isinstance(swipe, dict): + f_key = swipe.get('fieldKey') + left, right = swipe.get('leftOption'), swipe.get('rightOption') + ok = (f_key in valid_keys + and isinstance(left, str) and isinstance(right, str)) + if ok: + left, right = left.strip()[:MAX_SWIPE_OPTION], right.strip()[:MAX_SWIPE_OPTION] + # Both non-empty, and DISTINCT: one option on both sides is a deck whose two + # gestures do the same thing, which is two spellings of one state (the + # `kanbanClamp` law) wearing a control that promises a choice. + if left and right and left.casefold() != right.casefold(): + out['swipe'] = {'fieldKey': f_key, 'leftOption': left, 'rightOption': right} + # ── ⭐⭐ WAVE-37 T20 (owner item 10 / R2, contract C2): WHICH SCRIPT THIS VIEW RENDERS ── + # + # `{id}` — the row in E's per-database script store (`/api/v1/script-views`) whose source this + # view draws. WHOLE-KEY drop, the `swipe` construction rather than the `charts` one: a script + # view with no id is not a degraded script view, it is a panel with nothing to run, and the + # honest state for that is the unconfigured one the client already has a surface for. + # + # ⛔ THE ID IS DELIBERATELY NOT CHECKED AGAINST `valid_keys`, AND THAT IS NOT AN OVERSIGHT. + # `valid_keys` is this table's FIELD keys; a script id names a record in a DIFFERENT store that + # this function cannot reach. So the host bounds the shape and nothing else, and the renderer + # owns the one loss this is blind to — the script row was deleted — which it must SHOW rather + # than fall back to another script (the `viewModes.tsx` house rule `_clean_swipe`'s note cites). + # ⚠ Mirrored key-for-key by `customer-grid/types.ts::cleanDisplay`. That file rebuilds the + # config key by key on every autosave, so a key accepted HERE and unknown THERE is erased on the + # next column resize — which is the half of the round trip this docstring's W33 note warns about. + script = raw.get('script') + if isinstance(script, dict): + s_id = script.get('id') + if isinstance(s_id, str) and s_id.strip(): + out['script'] = {'id': s_id.strip()[:MAX_SCRIPT_ID]} + # ── ⭐⭐ WAVE-29 R7 (item 10): the FORM spec — see `_clean_form` for why the token is not here. + form = _clean_form(raw.get('form'), valid_keys) + if form: + out['form'] = form + # ── ⭐⭐ W33-T45 / CONTRACT C1 / RULING R5 (owner item 8b): IS THIS INTERFACE PUBLISHED ──── + # + # ⛔ TWO KEYS LIVE HERE AND TWO DELIBERATELY DO NOT. The `published` flag and the sharer's + # `public | password` choice are DISPLAY state — the view says what it is, the client renders + # a badge from it, and it travels with the view like every other key in this dict. The SECRET + # TOKEN and the PASSPHRASE HASH do not: they live in the server-only bucket, exactly as + # `routes_forms.py`'s `TOKENS_KEY` holds the form token. `config.display` is echoed back to + # every user who can open the view, so a token in here is a token published to the audience + # the password was meant to exclude. `_clean_form` above carries the same rule and the same + # reason; this is the second door, not a new one. + # + # ⛔ AND THE COERCION IS FAIL-CLOSED, WHICH IS WHY THIS IS NOT A BARE ALLOWLIST. A plain + # allowlist drops an unrecognised `publishAccess` and KEEPS `published: True` — leaving a + # published view with no stated access, i.e. a third state neither the ruling nor the client + # has a meaning for, on the one key where guessing wrong publishes a tenant's data to the + # open internet. So: a published view ALWAYS carries an access, and anything that is not the + # literal `'public'` reads as `'password'`. The unpublished case stores nothing at all — + # absent means unpublished, and a `published: False` would be the second way to say nothing + # that `kanbanClamp` and `mode: 'grid'` are both here to forbid. + if raw.get('published') is True: + out['published'] = True + out['publishAccess'] = 'public' if raw.get('publishAccess') == 'public' else 'password' + return out + + +#: FOLDERS over the saved views / cohorts sidebars (wave-8 I11, contract C4). +#: +#: ⚠ Folder membership is stored as a SIDE MAP (`itemFolders`), not as a `folderId` ON each +#: view or cohort — a deliberate amendment to C4's first wording, recorded in the split doc. +#: Two reasons. (1) A cohort lives in a DIFFERENT store (`customer_cohorts`, keyed by cohort +#: id) and adding a `folders` key beside those ids would collide with a cohort whose generated +#: id happened to be 'folders'. (2) Folder placement is a per-user ORGANISING act, not part of +#: what a view IS: keeping it out of the view config means duplicating or exporting a view does +#: not drag a folder reference along with it. One map, one home, both surfaces. +FOLDER_SURFACES = {"views", "cohorts"} +MAX_FOLDERS = 60 +MAX_FOLDER_NAME = 80 + +#: Wave-9 I15 (contract C5) — a user-chosen folder icon, as {shape, tone}. +#: +#: Both halves are WHITELISTS, never free values: `shape` names geometry the client already +#: draws (one source, `iconShapes.ts`, read by both painters) and `tone` names a palette token, +#: not a colour — so a browser cannot post a hex and defeat a tenant restyle, and a shape the +#: client cannot render can never reach the store. +#: ⚠ MIRRORS CLIENT'S `iconShapes.ts` ENUMERATION EXACTLY (C5: CLIENT enumerates, HOST mirrors — +#: posted in the split doc 2026-07-29, HOST adopted it the same day, replacing a provisional +#: 12-shape guess of mine that contained shapes the client cannot draw). Do not extend this set +#: without the matching client geometry: an unknown shape falls back to the default folder mark, +#: which is also I14's "existing folders get the folder icon" for every pre-wave-9 folder. +FOLDER_ICON_SHAPES = {"folder", "star", "flag", "tag", "bookmark", "box", "circle", "square"} +#: Tones are the C1 pastels — FILLS ONLY, never text (the standing palette rule). 'grey' is the +#: default, and is CLIENT's key name: not 'neutral', which is what HOST first guessed. +FOLDER_ICON_TONES = {"blue", "green", "yellow", "red", "grey"} +FOLDER_ICON_DEFAULT_TONE = "grey" + + +#: Wave-9 I17 (contract C4) — who may EDIT a saved view. +#: +#: ⚠ READ THIS BEFORE BUILDING ON IT. Views are stored PER USER today +#: (`core/table_store.TableStore.workspace` reads `store.get(table_key)[username]`), so one +#: user's views are invisible to every other user and "collaborative" has nothing to act on +#: yet. This validator is therefore CORRECT-BUT-INERT plumbing: it makes the setting durable +#: and fail-closed now, so that when shared views land the permission does not need a data +#: migration and no saved view is retro-restricted. It does NOT make anything shared, and +#: nothing in the app currently reads it to grant or deny cross-user access. +#: Recorded as the C4 amendment in .claude/wiki/research/grid-wave9-split.md. +VIEW_EDIT_MODES = {'personal', 'collaborative', 'users'} +MAX_VIEW_USERS = 50 + + +def clean_view_permissions(raw, default, known_users=None): + """{edit, users?} — fail-closed on both halves. + + `default` is supplied by the CALLER because it splits by path, and that split is a + permission rule rather than a formatting one: absent on a view that already exists means a + pre-wave-9 view and must stay 'collaborative' (retro-restricting somebody's saved view is a + silent takeaway), while absent on CREATE must be 'personal' (a new view must never be + anyone-can-edit purely by omission). + + `known_users` (when given) is the real account list: an unknown name is DROPPED, and an + 'users' grant left with nobody in it collapses to 'personal' rather than to everyone. + """ + mode = (raw or {}).get('edit') if isinstance(raw, dict) else None + if mode not in VIEW_EDIT_MODES: + mode = default if default in VIEW_EDIT_MODES else 'personal' + if mode != 'users': + return {'edit': mode} + names, seen = [], set() + for u in list((raw or {}).get('users') or [])[:MAX_VIEW_USERS]: + u = str(u or '').strip() + if not u or u.lower() in seen: + continue + if known_users is not None and u not in known_users: + continue # fail-closed: a name we cannot resolve grants nothing + seen.add(u.lower()) + names.append(u) + if not names: + return {'edit': 'personal'} # an empty grant is NOT "everyone" + return {'edit': 'users', 'users': names} + + +def clean_folder_icon(raw): + """{shape, tone} or None. Fail-closed on both halves, independently. + + A folder with a valid shape but a junk tone keeps the shape and defaults the tone rather + than losing the icon entirely — losing a user's pick because one half was wrong is the kind + of silent data loss the folder events already avoid elsewhere. + """ + if not isinstance(raw, dict): + return None + shape = raw.get("shape") + if shape not in FOLDER_ICON_SHAPES: + return None + tone = raw.get("tone") + return {"shape": shape, + "tone": tone if tone in FOLDER_ICON_TONES else FOLDER_ICON_DEFAULT_TONE} + + +def clean_folders(raw): + """Validate the per-surface folder lists, fail-closed. {surface: [{id, name, order}]}.""" + out = {} + for surface in FOLDER_SURFACES: + items, seen = [], set() + for f in list((raw or {}).get(surface) or [])[:MAX_FOLDERS]: + if not isinstance(f, dict): + continue + fid = str(f.get("id") or "").strip()[:80] + name = str(f.get("name") or "").strip()[:MAX_FOLDER_NAME] + if not fid or not name or fid in seen: + continue # an unidentified or unnamed folder cannot be shown or edited + seen.add(fid) + try: + order = int(f.get("order", len(items))) + except (TypeError, ValueError): + order = len(items) + row = {"id": fid, "name": name, "order": order} + icon = clean_folder_icon(f.get("icon")) # wave-9 I15 (C5); absent = default mark + if icon: + row["icon"] = icon + items.append(row) + items.sort(key=lambda x: x["order"]) + for i, f in enumerate(items): + f["order"] = i # re-index so `order` is always dense and total + if items: + out[surface] = items + return out + + +#: ⭐⭐ WAVE 32 · OWNER ITEM 20 (`W32-T27`, raised by SESSION C as ASK C-16) — "FILED AT ROOT". +#: +#: ⛔ THE DEFECT IS THAT ROOT WAS REPRESENTED BY *ABSENCE*, AND ABSENCE CANNOT HOLD TWO FACTS. +#: "this arrived by grant and was never filed" and "the receiver deliberately dragged this OUT of +#: the Shared group" were the same stored state — nothing — so the client had to GUESS, and +#: `folders.ts::groupByFolder` guessed "Shared". That is why only folder→folder moves appeared to +#: work: **the root bucket was unreachable for a shared view by construction.** +#: +#: ⚠ A RESERVED FOLDER ID, NOT A NEW FIELD, deliberately. The placement map is `{itemId: folderId}` +#: and every reader on both sides already understands it; a parallel "filedAtRoot" set would be a +#: second source of truth for one question, and the two would disagree the first time one of them +#: was written without the other. This id names no folder BY DESIGN and is therefore exempt from +#: the folder-exists test below — it is the one value that means "no folder, on purpose". +#: ⚠ Spelled `ROOT_FOLDER_ID` on the client (`customer-grid/folders.ts`, C's file). Two spellings +#: of one constant is [[a-constant-two-features-share]]; `verify_folders`/`verify_api` assert they +#: agree rather than a comment asking nicely. +ROOT_PLACEMENT = "__root__" + + +def clean_item_folders(raw, folders, valid_ids): + """{surface: {itemId: folderId}} — dropping any placement whose ITEM or FOLDER is gone. + + This is what makes a deleted folder's contents fall back to the root rather than vanish: + nothing stores "this item is in no folder", so an unresolvable placement simply disappears + and the item renders at the top level. Same for an item that was deleted elsewhere — its + stale placement can never resurrect it, because the sidebars render ITEMS and consult this + map, never the other way round. + + ⭐⭐ WAVE 32 — THE PARAGRAPH ABOVE STATES THE FEATURE AND THE BUG IN ONE SENTENCE, and it took + owner item 20 to notice they were the same mechanism. *"Nothing stores 'this item is in no + folder', so an unresolvable placement simply disappears"* is exactly right for a DELETED FOLDER + (its contents should fall to the root) and exactly wrong for a SHARED VIEW (falling back means + falling back INTO the Shared group, which is where it started). `ROOT_PLACEMENT` is the value + that survives this function so the second case can be said out loud. + """ + out = {} + for surface in FOLDER_SURFACES: + fids = {f["id"] for f in (folders or {}).get(surface, [])} + ok = {} + for item_id, fid in ((raw or {}).get(surface) or {}).items(): + if not isinstance(item_id, str) or not isinstance(fid, str): + continue + # ⛔ `fid == ROOT_PLACEMENT` FIRST, and it is NOT in `fids` — it names no folder, which + # is the whole point. Without this clause the value is written by `item_move` and + # scrubbed here on the way back out, so the mark would be stored and instantly lost: + # the two halves are ONE change and shipping either alone is worse than shipping + # neither ([[lost-write-looks-like-failed-read]]). + if item_id in (valid_ids or {}).get(surface, ()) and (fid == ROOT_PLACEMENT + or fid in fids): + ok[item_id[:120]] = fid + if ok: + out[surface] = ok + return out + + +def view_refs_of(nodes): + """C4 — every view id a tree REFERS to, deduped, in order. Mirrors `types.ts::viewRefsOf`. + + ⚠ An INACTIVE leaf (no view chosen yet) is not a reference, on both engines: counting one + would refuse a save the moment somebody opened the condition and had not picked anything. + """ + out = [] + for node in nodes or []: + if not isinstance(node, dict): + continue + if 'children' in node: + for vid in view_refs_of(node.get('children')): + if vid not in out: + out.append(vid) + continue + if node.get('colId') != VIEW_FIELD: + continue + vid = str(node.get('value') or '').strip()[:MAX_VIEW_ID] + if vid and vid not in out: + out.append(vid) + return out + + +def view_filter_cycle(view_id, nodes, filters_of): + """⭐⭐ W37-T26 / C4 — THE CYCLE REFUSAL's host half. Mirrors `types.ts::viewFilterCycle`. + + View X filtered on "is in View Y" where Y is filtered on X cannot be resolved by anything: + X needs Y needs X. Returns the CYCLE PATH (ids, starting and ending at `view_id`) or None. + + ⛔ THE CLIENT REFUSES THE SAME SHAPE AND THAT IS NOT DUPLICATION. The browser's guard is what + a PERSON sees and is the only one that can explain itself; this one is what makes the rule true + for a caller that POSTs a view directly, which is every guard-in-the-browser's blind spot. + ⚠ A view referring to ITSELF is a cycle of length one and is caught by the same walk — also + the case a person reaches most easily, by duplicating a view. + """ + def walk(current, path, seen): + refs = view_refs_of(nodes if current == view_id else filters_of(current)) + for nxt in refs: + if nxt == view_id: + return path + [nxt] + if nxt in seen: + continue + seen.add(nxt) + found = walk(nxt, path + [nxt], seen) + if found: + return found + return None + + return walk(view_id, [view_id], {view_id}) + + +def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None, + visible_view_ids=None): + """Recursively validate an UNTRUSTED filter tree (conditions + nested groups). + + Returns a clean tree of leaf conditions ({colId, op, value, value2}) and + groups ({conj, children}). Module-agnostic on purpose: any module embedding + the grid validates its own view state through this one function. + + Fail-closed PER NODE: anything unrecognised is DROPPED rather than raised — + the same contract the rest of the view sanitiser follows, so one bad rule can + never cost a user their whole saved view. Depth, per-level width and total + node count are all capped: the tree is re-evaluated for every row on every + render, so an unbounded structure would be a persistent client-side DoS. + Empty groups are dropped (they carry no meaning once persisted). + + `cohort_ids` is the set of cohorts the CALLER may see. A cohort leaf naming anything else is + DROPPED here rather than left for the engine — a deleted cohort would otherwise leave a + condition that can only match nothing, so `List is not [deleted]` would show an empty table + forever with no way to tell why. `None` means this host has no cohorts, and then every + cohort leaf is dropped: fail-closed, like every other unknown key. + + ⭐⭐ W37-T26 (C4) — `visible_view_ids` is the same parameter one leaf over: the views this + caller may name in a `__view__` membership leaf. A leaf naming anything else is DROPPED here + for the identical reason, and `None` (this host has no views to offer) drops every one. + ⚠ DEFAULTS TO `None` SO EVERY EXISTING CALLER IS BYTE-IDENTICAL. Three callers pass nothing + (`routes_admin`, `routes_query` twice) and each of them is a surface with no saved-view + vocabulary of its own; under the fail-closed rule they drop the leaf, which is correct rather + than merely safe. + """ + if budget is None: + budget = [MAX_FILTER_NODES] + out = [] + for node in list(raw or [])[:MAX_FILTER_SIBLINGS]: + if budget[0] <= 0: + break + if not isinstance(node, dict): + continue + if isinstance(node.get('children'), list): # a condition GROUP + if depth >= MAX_FILTER_DEPTH: + continue # too deep -> drop + budget[0] -= 1 + children = clean_filter_tree(node['children'], valid_keys, + depth + 1, budget, cohort_ids, visible_view_ids) + if children: + out.append({'conj': 'or' if node.get('conj') == 'or' else 'and', + 'children': children}) + continue + if node.get('colId') == COHORT_FIELD: # a cohort-membership leaf + op = COHORT_OP_ALIASES.get(node.get('op'), node.get('op')) + named = parse_cohort_ids(node.get('value')) + # ALL of them, or the leaf goes. A set that quietly lost a member asks a DIFFERENT + # question, and for `noneOf` a strictly wider one: `is none of [A, B]` degrading to + # `is none of [A]` would show every row in B under a count nobody would doubt. This + # is the same all-or-nothing the single-cohort leaf already had, extended to a set. + if op in COHORT_OPS and named and all(c in (cohort_ids or ()) for c in named): + budget[0] -= 1 + out.append({'colId': COHORT_FIELD, 'op': op, + 'value': ','.join(named), 'value2': ''}) + continue + if node.get('colId') == VIEW_FIELD: # C4: a view-membership leaf + # ⛔ THE SAME ALL-OR-NOTHING THE COHORT ARM ABOVE APPLIES, AND FOR THE SAME REASON. + # A leaf naming a view this caller cannot see is DROPPED rather than kept, because a + # kept one can only ever match nothing: `View is not [a view you cannot see]` would + # show an empty table forever with no way to tell why. Dropping it puts the view back + # in its honest unfiltered state, which is what `clean_filter_tree`'s own docstring + # already promises for cohorts. + # ⚠ `visible_views` is what the caller passes; `None` means this host does not know, + # and then EVERY view leaf is dropped — fail-closed, like every other unknown key. + v_op = node.get('op') + v_id = str(node.get('value') or '').strip()[:MAX_VIEW_ID] + if v_op in VIEW_OPS and v_id and v_id in (visible_view_ids or ()): + budget[0] -= 1 + out.append({'colId': VIEW_FIELD, 'op': v_op, 'value': v_id, 'value2': ''}) + continue + if node.get('colId') in valid_keys and node.get('op') in FILTER_OPS: + budget[0] -= 1 + # `or ''` would be wrong here: it maps every FALSY value to '', and '' is the + # signal for "inactive". A numeric 0 (or 0.0, or False) is a real value the client + # treats as active — `0 === ""` is false in TS — so `revenue = 0` would silently + # stop filtering and show every row instead of the zero-revenue ones. + val, val2 = node.get('value'), node.get('value2') + leaf = {'colId': node['colId'], 'op': node['op'], + 'value': ('' if val is None else str(val))[:500], + 'value2': ('' if val2 is None else str(val2))[:500]} + # CG-8. A MEASURE condition ("Sales, in the last 90 days, > 5,000") carries two + # extra members: a stable client-generated `id`, which is how the server's answer + # finds its way back to the condition that asked (positional matching silently + # re-associates every answer the moment a user deletes a condition), and the + # `window`. Emitted ONLY when the input has them — a column condition's cleaned + # shape is unchanged, so every persisted view deserialises byte-identically and + # `clean_filter_tree` stays idempotent (verify_filter_engine.py asserts that by + # exact structural comparison). + rid = node.get('id') + if rid not in (None, ''): + leaf['id'] = str(rid)[:64] + window = _clean_window(node.get('window')) + if window is not None: + leaf['window'] = window + # Owner items 3 + 4, carried under the SAME rule as CG-8's `id`/`window`: emitted + # only when the input has them, so a plain column condition's cleaned shape is + # byte-identical to what it always was and `clean_filter_tree` stays idempotent + # (verify_filter_engine.py asserts that by exact structural comparison). Drop the + # carry-through and the next autosave silently strips a date condition back to a + # bare comparison against an empty value — i.e. back to INACTIVE. + date_window = _clean_window(node.get('dateWindow')) + if date_window is not None: + leaf['dateWindow'] = date_window + mode = node.get('dateMode') + # SHAPE only. An unrecognised mode survives here and is refused by + # `windows.resolve_anchor`, which returns None and makes the condition match + # NOTHING — the same split as `_clean_window`, and the reason this module can stay + # free of the date vocabulary it would otherwise have to keep in step. + if isinstance(mode, str) and 0 < len(mode) <= 40: + leaf['dateMode'] = mode + rhs = _clean_rhs(node.get('rhs'), valid_keys) + if rhs is not None: + leaf['rhs'] = rhs + out.append(leaf) + return out + + +def _default_view_config(fields): + # ⭐⭐ W30-T41's SERVER HALF (F's ask F-1, answered by D — this file is D's fence). + # + # ⛔ THE SECOND ARM USED TO BE `or field["source"] == "overlay"`, AND IT SWALLOWED THE FIRST + # ONE FOR EVERY CONNECTED COLUMN. `user_tables._clean_field` stamps `source: "overlay"` on + # every `ut_` field, so on an Odoo grid the arm was true for ALL of them and `default: False` + # meant nothing: `odoo_id`, `state`, `customer_link` and `partner_id` opened SHOWN however + # they were declared. The exception had become the rule ([[fallback-that-became-the-rule]]), + # and it is the same predicate `useGridColumns.isDefaultVisible` carried on the client. + # + # ⚠ AND THE TWO HALVES MUST MOVE TOGETHER, which is why this is not cosmetic. `CustomerGrid` + # compares the stored view against its own `defaultViewConfig` by JSON equality; with the + # client fixed (T41) and this left alone, the system view would differ from the client's + # default on every render — a view that looks permanently dirty and autosaves forever, which + # is the failure `verify_filter_engine`'s key-ORDER check exists to prevent, one level down. + # + # ⚠ A USER-CREATED COLUMN IS UNAFFECTED, and that is why the fix is a DELETION rather than a + # carve-out for the four Odoo keys: it carries no `default` key at all, so `is not False` + # keeps it visible. On the main Customer grid exactly one field moves — `notes`, which asks + # to be hidden in its own declaration and was being shown against it. + shown = [field["key"] for field in fields if field.get("default") is not False] + hidden = [field["key"] for field in fields if field["key"] not in shown] + return { + # `filters` is the ROOT of the filter tree: leaf conditions and/or nested + # condition groups ({conj, children}); `filterConj` joins the root level. + "filters": [], "filterConj": "and", + "sorts": [], "groupBy": None, "colorBy": None, + "rowHeightMode": "short", "order": shown + hidden, "visible": shown, + "widths": {}, "memberPids": [], + } + + +#: Wave 17 R1 / C-LOCKV — the `kind` a PROJECTED locked view wears. A cohort is not a separate +#: kind of object any more: it is a saved view whose rows are a hand-curated set. +LOCKED_VIEW_KIND = 'locked' + + +def locked_view_projection(entry, base_config): + """One cohort -> the saved-view row that IS it (wave 17 R1, contract C-LOCKV). + + ⛔ THE LOCK IS THE VIEW'S IDENTITY, NOT ITS CONFIGURATION. `config.cohortLock` names the + view's OWN id, which is what makes the shipped engine law (`useVisibleRows`: intersect the + named set FIRST, unconditionally, and match NOTHING when the membership is unresolvable) do + all the work with no second mechanism. Membership itself is NEVER copied in here — it stays + in `customer_cohorts` and travels as `workspace.lists`, because a per-reader-scoped + collection inside a client-writable `config` is deleted by the next autosave (see the + contract's reason 2). + + ⚠ `locked: True` is the LEGACY "undeletable/mode-frozen" flag and is deliberately NOT set: + these views are ordinary in every respect the owner asked for — reorder, folder, sort, + filter, change display mode. The lock mark in the rail is driven by `kind`. + """ + return { + 'id': entry['id'], + 'name': entry.get('name') or entry['id'], + 'kind': LOCKED_VIEW_KIND, + 'config': {**base_config, 'cohortLock': entry['id']}, + } + + +#: ⭐ WAVE-27 item 27 (owner ruling R8) — the IG "Overview" view's CURATED COLUMNS, in the +#: owner's own order: handle, followers, engagement, location, last enriched. +#: +#: Written as candidates rather than as a requirement. The template registry REFUSES a template +#: whose columns the target lacks (`view_templates.missing_columns`) because applying one writes +#: the user's own views and a filter on a missing column silently WIDENS. This view is INJECTED, +#: not applied, and it filters nothing — so the proportionate rule is the opposite one: take the +#: columns the table has, in this order, and skip the rest. An IG database that predates a +#: column simply shows the other four. +#: +#: ⚠ `location_guess` is SESSION B's item-16 column and may not exist yet. That is exactly why +#: this list is intersected rather than asserted: a hard requirement here would make the whole +#: view vanish (or the assembly refuse) on every tenant until B lands, and then appear by +#: surprise. `profile_url` closes the list as the click-through, which is what makes the view +#: usable rather than merely informative. +IG_OVERVIEW_COLUMNS = ('handle', 'full_name', 'followers', 'avg_engagement', + 'location_guess', 'enriched_at', 'profile_url') + +#: The id is PINNED, the `view_templates` discipline: re-assembling must update the same view +#: rather than mint "Overview 2". It also lets a user's own edits overlay it through the saved +#: -config loop below, exactly as a cohort projection does. +IG_OVERVIEW_ID = 'tpl_overview' + +#: How this function recognises an IG preset database WITHOUT importing the engine: two of the +#: profile preset columns is a stronger signal than any single one (a hand-made table could +#: plausibly own a column called `followers`; owning `followers` AND `avg_engagement` AND +#: `handle` is the preset set). `core/` must stay importable without the API layer, so this +#: mirrors `user_tables.PROFILE_PRESET_KEYS` the way that module mirrors the engine's. +_IG_SIGNATURE = ('handle', 'followers', 'avg_engagement') + + +def _ig_overview_view(fields, base): + """R8's curated Overview, or None when this table is not an Instagram one.""" + keys = {f['key'] for f in fields} + if not all(k in keys for k in _IG_SIGNATURE): + return None + visible = [k for k in IG_OVERVIEW_COLUMNS if k in keys] + return { + 'id': IG_OVERVIEW_ID, + 'name': 'Overview', + 'kind': 'system', + # NOT `locked`. The system view is locked because it is the identity of the table ("show + # me everything"); this one is a STARTING LAYOUT, and R8 calls it curated rather than + # fixed. A user who wants a sixth column should get one. + 'note': 'The five things worth seeing first on a creator. Sorted by reach.', + 'config': { + **dict(base), + 'visible': visible, + 'order': visible + [k for k in (f['key'] for f in fields) if k not in visible], + 'sorts': ([{'colId': 'followers', 'dir': 'desc'}] + if 'followers' in keys else []), + }, + } + + +def views_from_defs(defs, saved_views, fields, system_name="All customers", locked_lists=None, + view_order=None): + """Convert legacy list formulas into the shared serializable SavedView contract. + + `system_name` (wave 16 C-TOPIC) labels the system view per TOPIC ("All products" on the + product surface). The ID stays "all-customers" on every topic — the client pins it + (UNDELETABLE_VIEW_IDS, the landing default), and an id that varies by surface would fork + that contract for a label's sake. + + `locked_lists` (wave 17 R1) are the caller's cohorts, each PROJECTED as a saved view whose + id IS the cohort id — so every stored reference to that id (a `cohortLock` on another view, + an `is part of` condition, a folder placement) keeps pointing at the same thing and no + rewrite map is needed. Saved config OVERLAYS the projection through the same mechanism the + `list:` views have always used, which is what gives a locked view its own sort, filter, + columns and display mode with no new storage.""" + base = _default_view_config(fields) + views = [{ + "id": "all-customers", "name": system_name, "kind": "system", + "locked": True, "config": dict(base), + }] + # ⭐ WAVE-27 item 27 (R8) — the IG Overview, ABOVE All records. + # + # ⚠ INJECTED, not seeded into the store, and that is what makes "existing IG databases gain + # it too" true with no migration and no write on a read path. It is the same mechanism the + # system view above has always used; the pinned id means a user's own edits overlay it + # through the saved-config loop below rather than forking a second view. + _overview = _ig_overview_view(fields, base) + if _overview: + views.insert(0, _overview) + op_map = {">=": "gte", ">": "gt", "<=": "lte", "<": "lt", "=": "eq", + "contains": "contains"} + for name, definition in (defs or {}).items(): + filters = [] + for rule in definition.get("rules") or []: + if rule.get("field") not in {field["key"] for field in fields}: + continue + filters.append({ + "colId": rule["field"], + "op": op_map.get(rule.get("op"), "eq"), + "value": str(rule.get("value") if rule.get("value") is not None else ""), + }) + sort = str(definition.get("sort") or "") + sorts = ([{"colId": sort.lstrip("-"), + "dir": "desc" if sort.startswith("-") else "asc"}] + if sort.lstrip("-") in {field["key"] for field in fields} else []) + views.append({ + "id": "list:" + str(name), + "name": str(name), + "kind": "list", + "note": str(definition.get("note") or ""), + "config": { + **base, "filters": filters, "sorts": sorts, + "memberPids": [int(pid) for pid in definition.get("members") or [] + if isinstance(pid, int) or str(pid).isdigit()], + }, + }) + # Wave 17 R1 — the cohorts, as ordinary views. Appended BEFORE the saved-config overlay + # below so a user's own edits to a locked view (its sort, its columns, its display mode) + # land on the projection instead of creating a second row with the same id. + for _entry in (locked_lists or []): + if isinstance(_entry, dict) and _entry.get('id'): + views.append(locked_view_projection(_entry, base)) + index = {view["id"]: i for i, view in enumerate(views)} + for view_id, saved in (saved_views or {}).items(): + if not isinstance(saved, dict) or not isinstance(saved.get("config"), dict): + continue + clean = dict(saved) + clean["id"] = str(view_id) + if view_id in index: + views[index[view_id]] = clean + else: + views.append(clean) + # ⛔ WAVE 17 R1 — RE-STAMP THE LOCK AFTER THE OVERLAY. The loop above REPLACES a projected + # view with its saved record, and a saved record that omits `cohortLock` would hand back a + # view that shows the WHOLE BOOK under a locked view's name. That is not hypothetical: the + # client rebuilds `config` on every autosave (a column resize is enough), and the lock is + # identity here, not something the browser is the source of truth for. Read-side rather than + # write-side-only on purpose — this also repairs any record already written by another path. + _locked_ids = {e['id']: e for e in (locked_lists or []) + if isinstance(e, dict) and e.get('id')} + if _locked_ids: + for _v in views: + _entry = _locked_ids.get(_v.get('id')) + if not _entry: + continue + _v['kind'] = LOCKED_VIEW_KIND + _v['config'] = {**(_v.get('config') or {}), 'cohortLock': _v['id']} + # One thing, one name: the cohort store owns it (the rename event routes there), so + # a stale `name` on the saved record can never fork into a second title. + _v['name'] = _entry.get('name') or _v['id'] + # ── ⭐ WAVE-27 item 5, contract C7: the PER-USER VIEW ORDER ─────────────────────────────── + # + # `view_order` is a list of view ids this user dragged into place. Applied LAST, over the + # finished list, so it reorders whatever the assembly produced without having to know how any + # of it got there (system, list:, cohort projection, saved, injected Overview). + # + # ⛔ THE SYSTEM VIEW STAYS AT INDEX 0 (C7), and it is re-pinned here rather than trusted to + # sort correctly: `all-customers` is the client's landing default and one of its + # UNDELETABLE_VIEW_IDS, so a stored order that happened to omit it — or list it third — + # would move the rail's home row. ⚠ R8's Overview is the ONE thing allowed above it, because + # the owner put it there; it is re-pinned with the system view so a drag cannot bury it + # either. Both are facts about the table rather than the user's arrangement of it. + # + # ⚠ UNKNOWN IDS APPEND IN SERVER ORDER (C7). A view created since this order was stored, or + # one shared to this user yesterday, must APPEAR — dropping it would make sharing look + # broken, and the failure would be invisible to whoever shared it. Ids in the stored order + # that no longer resolve are simply skipped. + if view_order: + _rank = {vid: i for i, vid in enumerate(view_order) if isinstance(vid, str)} + _pinned = [v for v in views if v.get('id') in (IG_OVERVIEW_ID, 'all-customers')] + _rest = [v for v in views if v.get('id') not in (IG_OVERVIEW_ID, 'all-customers')] + # A stable sort over a rank that DEFAULTS TO THE END keeps unranked views in their + # server order behind the ranked ones, rather than interleaving them by accident. + _rest.sort(key=lambda v: _rank.get(v.get('id'), len(_rank) + 1)) + views = _pinned + _rest + return views + + +def workspace_wire(ws, uname, pool_pids, defs=None, scope_key='customer', storage_key=None, + fields_base=None, with_cohorts=True): + """The client's `GridWorkspace` WIRE SHAPE from the stored table workspace — the ONE + projection, shared by both servers (app.py's `_table_grid` and the API's `/workspace`). + + ⛔ WHY THIS EXISTS (2026-07-30). The API route used to return the STORE shape with no + `storageKey` — and the client validator (`fetchWorkspace`) requires one, so the standalone + shell silently discarded the whole workspace: saved views never rendered and `cohortMode` + never arrived (the Cohort route drew the Customer surface). Duplicating the host's inline + projection into the route would have re-created the same drift one wave later; extracting it + means the wire can only be one thing. + + Returns `(workspace, fields, views, cohort_lists)` — the extra three because the host + interleaves further work (docs, derived cells, measure sets) that consumes them. + + HOST-ONLY extras stay with the host: `docs`/`docPayload`, `pool`, `hideViews`, + `cohortMode`/`scopeChoice` (the API stamps its own from `?scope=`). + + Wave 16 C-TOPIC: `fields_base` selects the canonical contract (absent = customer, + byte-identical). + + ⭐ WAVE 19 / R9 — `with_cohorts` NO LONGER MEANS "customer only". Wave 16 set it False on the + product surface because cohorts were a single customer-keyed bucket, so resolving them against + product pids would have intersected two unrelated id spaces and printed a plausible, + meaningless member count. `modules.cohort` is scope-parameterized now: the lists come from + THIS topic's bucket (`cohort_mod.scoped(scope_key)`), so their ids are this topic's ids and + the intersection with `pool_pids` is the ordinary one. The flag survives as an honest OFF + switch for a surface that wants no membership channel at all — it is not a topic wall. + """ + import modules.cohort as cohort_mod + + cohort_lists = [] + if with_cohorts: + for cid, c in sorted(cohort_mod.scoped(scope_key).visible(uname, pool_pids).items(), + key=lambda kv: (kv[1].get('name') or '').lower()): + members = [p for p in (c.get('members') or []) if p in pool_pids] + entry = {'id': cid, 'name': c.get('name') or cid, 'pids': members} + # Rule 8b: a member can drop out of the 24-month pool without the cohort being + # wrong, and a silently smaller cohort is exactly what the unverifiable-count rule + # forbids. + missing = len(c.get('members') or []) - len(members) + if missing: + entry['missing'] = missing + cohort_lists.append(entry) + + fields = fields_from_workspace(ws, cohorts=bool(cohort_lists), scope_key=scope_key, + fields_base=fields_base) + views = views_from_defs(defs or {}, ws.get('views'), fields, + # Wave 21 (item 3, R6): the system view's name is TOPIC-DERIVED. A + # user database's default view used to read "All customers" — a + # compiled customer literal minted on every topic, one half of the + # owner's "my new database looks like RI's Customer table". The ID + # stays 'all-customers' everywhere (pinned client+server — the + # client's UNDELETABLE set and the view pin both name it). + system_name=("All products" if scope_key == 'product' + else "All records" + if str(scope_key or '').startswith('ut_') + else "All customers"), + locked_lists=cohort_lists, + # ⭐ WAVE-27 item 5 (C7) — this user's own rail arrangement, from + # their own stratum. Read here rather than sorted by the client so + # the ORDER a request answers with is the order that was stored: + # sorting client-side would make the rail settle after a paint on + # every load, and shared views would land in server order first. + view_order=ws.get('viewOrder')) + workspace = {'storageKey': storage_key, 'views': views, 'lists': cohort_lists} + # Owner item 3 (2026-07-31): where this user left off. The client's own localStorage copy + # wins when present; this is the server's answer for a FRESH browser, which used to fall + # all the way to the system default view (and whatever display mode was stored on it). + if ws.get('activeViewId'): + workspace['activeViewId'] = str(ws['activeViewId']) + + # FOLDERS (owner item 11, contract C4), re-validated at SERVE time: a view or cohort can be + # deleted by a path that knows nothing about folders, and the placement map must not outlive + # the thing it points at. + _folders = clean_folders(ws.get('folders')) + _view_ids = {v['id'] for v in (views or []) if isinstance(v, dict) and v.get('id')} + # ── WAVE 17 R1 (C-LOCKV amendment 2026-08-03): the two folder surfaces become ONE, AT + # SERVE TIME rather than by a store migration. A locked view is an ordinary view now, so + # its folder has to be an ordinary view folder — but rewriting the stored map would be a + # one-shot write that has to be got right once, while this is a projection that is right + # every time it runs. New drags write to `views` anyway (the client only knows that + # surface), so `cohorts` drains on its own and never needs a second pass. + # ⚠ A cohorts-surface folder whose id ALREADY names a views folder is DROPPED, not merged: + # re-parenting somebody's list into a folder that merely shares an id is a worse outcome + # than the list appearing at the root, where it is visible and one drag from home. + _cf = list(_folders.get('cohorts') or []) + if _cf: + _vf = list(_folders.get('views') or []) + _taken = {f['id'] for f in _vf} + _order = len(_vf) + for _f in _cf: + if _f['id'] in _taken: + continue + _vf.append({**_f, 'order': _order}) + _order += 1 + _folders['views'] = _vf + _raw_item_folders = dict(ws.get('itemFolders') or {}) + if _raw_item_folders.get('cohorts'): + # Cohort placements now describe VIEWS (same ids — that is the point of preserving them). + # A placement already stored on the views surface WINS: it is the more recent act. + _raw_item_folders['views'] = {**dict(_raw_item_folders.get('cohorts') or {}), + **dict(_raw_item_folders.get('views') or {})} + _placed = clean_item_folders( + _raw_item_folders, _folders, + {'views': _view_ids, 'cohorts': {c['id'] for c in cohort_lists}}) + if _folders.get('views'): + workspace['folders'] = _folders['views'] + # ⛔ `cohortFolders` IS NO LONGER EMITTED. The rail has no cohorts section to fold, and a + # wire that still described one would invite a second rendering of rows that are now views. + _vplaced = _placed.get('views') or {} + for _v in (views or []): + if isinstance(_v, dict) and _v.get('id') in _vplaced: + _v['folderId'] = _vplaced[_v['id']] + _cplaced = _placed.get('cohorts') or {} + for _c in cohort_lists: + if _c['id'] in _cplaced: + _c['folderId'] = _cplaced[_c['id']] + + # RECORD LAYOUT (wave 2026-08-02, C-LAYOUT): the per-user record-detail field order, + # re-validated at SERVE time exactly like folders — a field can be deleted by a path + # that knows nothing about this stratum, and a stale key must not outlive its field. + _rl = ws.get('recordLayout') + if isinstance(_rl, dict) and isinstance(_rl.get('order'), list): + _fkeys = {f['key'] for f in fields if isinstance(f, dict) and f.get('key')} + _order, _seen = [], set() + for _k in _rl['order'][:200]: + _k = str(_k or '') + if _k and _k in _fkeys and _k not in _seen: + _seen.add(_k) + _order.append(_k) + if _order: + workspace['recordLayout'] = {'order': _order} + + return workspace, fields, views, cohort_lists + + +def embed_html_path(): + """The first existing candidate path for the inlined single-file build, or None.""" + for p in _EMBED_CANDIDATES: + if p.is_file(): + return p + return None + + +def scope_counts(shown, matched, total): + """The honest 'N of M' a SERVER-WINDOWED table must carry (CG-2). + + `matched` and `total` MUST come from their own queries over the whole scope. Never pass + `len(rows)` as `matched` — that is the silent [:N] this exists to prevent: the page would + report the window size as though it were the result size. + + Refuses the shapes that could only be a mistake, because a wrong count here is invisible on + screen (it looks like a smaller dataset, not like an error). + """ + shown, matched, total = int(shown), int(matched), int(total) + if matched > total: + raise ValueError(f"matched ({matched}) exceeds total ({total}) — a filter cannot match " + f"more rows than the scope holds") + if shown > matched: + raise ValueError(f"shown ({shown}) exceeds matched ({matched}) — the window cannot hold " + f"more rows than the filter matched") + return {"shown": shown, "matched": matched, "total": total, "windowed": True} + + + +# ⛔ EXIT-6 (2026-08-04): `build_html`, `component_dir`, `render` and `_DECLARED_COMPONENTS` WERE +# HERE, and they are gone with Streamlit. They were the EMBED HOST — the path that declared the +# prebuilt bundle as a `streamlit.components.v1` custom component (or injected the single-file +# HTML build as a fallback) so the React grid could be drawn inside a Streamlit page. +# +# THIS MODULE ITSELF SURVIVES, and that distinction is the whole point: `aios_grid.py` is imported +# at 11 sites across `aios-web/api/` plus `harness/semantic.py` — it owns the canonical field +# contract, the workspace wire and the count envelope. Only the ~95 lines that knew about a HOST +# went; the rest never did. Its one and only `import streamlit` lived inside `render`, lazily, and +# left with it. `api/verify_no_streamlit.py` now gates that nothing here re-imports it. +# +# Deleted with them: `aios_grid_embed.html` + `aios_grid_component/index.html` (a 2.1 MB prebuilt +# bundle), `build_embed.py` that produced them, and `deploy_hf.py`'s embed-staleness guard. The +# React app is now served directly by the FastAPI container — there is no twin to keep fresh, so +# the entire class of "the code shipped but the bundle did not" is retired rather than guarded. diff --git a/platform/aios_grid_fields.json b/platform/aios_grid_fields.json index 939c5e46e412410ccf1e9ee9d822bc261382a5af..3662c13e1343ae928010f70c2484164851afa9ac 100644 --- a/platform/aios_grid_fields.json +++ b/platform/aios_grid_fields.json @@ -1,581 +1,581 @@ -{ - "_comment": "CANONICAL field contract for the AIOS Airtable-style grid — the SINGLE source of truth. Consumed by platform/aios_grid.py (embed/Space host) and aios-web/api/main.py (standalone API), and regenerated into aios-web/web/public/sample_customers.json. Edit HERE only, then run aios-web/verify_fields_contract.py. source=odoo is READ-ONLY; source=overlay is the editable stratum (notes/tags) outside Odoo. type in {text,status,select,currency,int,date,pct} (select = a fixed-choice READ-ONLY brand attribute; dba is the first, wave 2026-08-02). `description` (wave 5) is the CANONICAL per-field description — every field must carry one, and since wave 7 (owner W8, 2026-07-28) every description is ONE SHORT PLAIN sentence (two only when a fact would otherwise mislead): what the field IS, nothing else — no filter tips, no '(none)' coaching, no rationale; the user's workspace NOTE overrides it in the (i) hover, never in this file. BUILDER FACT (documented here, deliberately NOT in user-facing text): blank text attributes display as '(none)', so `is '(none)'` — not `is empty` — finds the blanks on agent/city/state/country/zip/payment_terms/pricelist/tags. filterable:false = the CONDITION BUILDER does not offer it (still displayed, still sortable); every such field must have a replacement declared in aios-web/verify_fields_contract.py. 2026-07-27 partner attributes: country/zip/payment_terms/pricelist/tags/customer_since all ship default:false. zip is TEXT because a postal code has leading zeros. Odoo's credit_limit (1% populated) and user_id salesperson (2%) are deliberately ABSENT; agent_ids is the salesperson field and AR is where credit exposure comes from. Wave-5 item 8 (2026-07-27): ltm_rev and at_risk are DELETED — LTM's replacement is a creatable Sales measure column (the demo column IS Sales · the last 12 months), at_risk's replacement is a formula field, e.g. MAX(0, {revenue_ly} - {revenue_ytd}). Wave-6 item 8 (2026-07-27, the no-buildable-presets rule): revenue_ytd, revenue_ly, orders_24m, aov and yoy_pct are DELETED — every one is self-buildable, so a frozen pre-set beside the builder was two ways to ask one question. Replacements (recorded in verify_fields_contract.py): creatable measure columns for Sales / Orders / Avg order $ over any period (harness/measure_filter.py ADMITTED carries revenue, orders and the composite aov), and a formula over two measure columns for YoY, e.g. ({sales_ytd} - {sales_ly}) / {sales_ly}. Stale view colIds naming the five self-heal on the next autosave (the established rule).", - "fields": [ - { - "key": "customer", - "label": "Customer", - "type": "text", - "source": "odoo", - "pinned": true, - "default": true, - "description": "The customer's name in Odoo. One row per customer who ordered in the last 24 months." - }, - { - "key": "partner_id", - "label": "Odoo ID", - "type": "int", - "source": "odoo", - "derived": true, - "default": false, - "description": "The Odoo res.partner id — the key every Odoo document joins on. DERIVED: this row's pid IS the partner id, so a stored copy would be a second source." - }, - { - "key": "odoo_status", - "label": "Odoo record", - "type": "status", - "source": "odoo", - "default": false, - "options": [ - "Active", - "Archived" - ], - "description": "Whether this customer still exists in Odoo. Archived means deleted there." - }, - { - "key": "agent", - "label": "Agent", - "type": "text", - "source": "odoo", - "default": true, - "description": "The sales agent who owns this account." - }, - { - "key": "dba", - "label": "DBA", - "type": "select", - "source": "odoo", - "default": false, - "options": [ - "Fisch", - "Royal", - "Both" - ], - "description": "The brand this customer buys from - Fisch, Royal, or both. Amazon-channel orders are not a DBA." - }, - { - "key": "salesperson", - "label": "Salesperson", - "type": "text", - "source": "odoo", - "default": false, - "description": "Who keyed in most of this customer's orders — not the Agent, who owns the account." - }, - { - "key": "street", - "label": "Street", - "type": "text", - "source": "odoo", - "default": false, - "description": "First address line, from res.partner directly - not the geocoder, so a customer the map cannot place still shows its address." - }, - { - "key": "street2", - "label": "Street 2", - "type": "text", - "source": "odoo", - "default": false, - "description": "Second address line (suite, unit, floor) on the customer's Odoo address." - }, - { - "key": "city", - "label": "City", - "type": "text", - "source": "odoo", - "default": true, - "description": "City on the customer's Odoo address." - }, - { - "key": "state", - "label": "State", - "type": "text", - "source": "odoo", - "default": true, - "description": "State or province on the customer's Odoo address." - }, - { - "key": "country", - "label": "Country", - "type": "text", - "source": "odoo", - "default": false, - "description": "Country on the customer's Odoo address." - }, - { - "key": "zip", - "label": "ZIP", - "type": "text", - "source": "odoo", - "default": false, - "description": "Postal code on the customer's Odoo address." - }, - { - "key": "customer_since", - "label": "Customer since", - "type": "date", - "source": "odoo", - "default": false, - "description": "When this customer was first set up in Odoo." - }, - { - "key": "tags", - "label": "Tags", - "type": "text", - "source": "odoo", - "default": false, - "description": "Odoo labels on this customer, comma-separated." - }, - { - "key": "pricelist", - "label": "Price list", - "type": "text", - "source": "odoo", - "default": false, - "description": "The price list this customer buys on." - }, - { - "key": "payment_terms", - "label": "Payment terms", - "type": "text", - "source": "odoo", - "default": false, - "description": "Payment terms on this customer's account — Net 30, for example." - }, - { - "key": "last_order", - "label": "Last order", - "type": "date", - "source": "odoo", - "default": true, - "description": "Date of the most recent confirmed order." - }, - { - "key": "overdue_days", - "label": "Overdue days", - "type": "int", - "source": "odoo", - "default": true, - "description": "How many days late this customer is running against their own usual ordering rhythm." - }, - { - "_note": "filterable:false — DERIVED ANALYTIC: est_missed is min(cycles missed, 3) x AOV, a score we compute rather than an object the business has, so a condition on it would read as a fact about the customer when it is a fact about our arithmetic. It still displays and still sorts. Until wave 6 this flag also covered the frozen-window presets (revenue_ytd / revenue_ly / orders_24m / aov / yoy_pct); those are now DELETED outright under the owner's no-buildable-presets rule — see _comment. est_missed itself STAYS: no creatable measure or formula reproduces the cadence model behind it.", - "key": "est_missed", - "label": "Est. missed $", - "type": "currency", - "source": "odoo", - "default": true, - "agg": "sum", - "filterable": false, - "description": "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed." - }, - { - "_note": "wave 21 R1 — KEY UNCHANGED, LABEL RENAMED. The computation is a DISJOINT split (ar.py credit_exposure): this column is only the not-yet-due residual, its sibling is the past-grace residual, and the two sum to the total. Under the label 'AR open $' the majority-late book read as 'Overdue > Open', which is nonsense in AR vocabulary — 'open' universally means the total. The label now says what the number is; the key stays so saved views and filters keep working.", - "key": "ar_open", - "label": "AR current $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "Invoiced money owed but not yet due (a 5-day grace applies before it counts as overdue)." - }, - { - "key": "ar_overdue", - "label": "AR overdue $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "Invoiced money past due — same basis as the Collections page." - }, - { - "_note": "wave 21 R1 — the TOTAL, added beside the rename above. AR current $ + AR overdue $, i.e. what most people mean by 'open AR'. Composed from the same ar.credit_exposure rows the siblings use, so it is transitively reconciled by ar.validate()'s residual read_group tie — no second oracle.", - "key": "ar_outstanding", - "label": "AR outstanding $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "Total invoiced money owed right now: AR current $ plus AR overdue $." - }, - { - "key": "ar_exposure", - "label": "Credit exposure $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "The most you could be out if they stopped paying today: open, overdue, draft and not-yet-invoiced." - }, - { - "key": "ar_aged_1_30", - "label": "1-30 days $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $." - }, - { - "key": "ar_aged_31_60", - "label": "31-60 days $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $." - }, - { - "key": "ar_aged_61_90", - "label": "61-90 days $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $." - }, - { - "key": "ar_aged_90_plus", - "label": "90+ days $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "Overdue by more than 90 days. The four aging buckets sum to AR overdue $." - }, - { - "key": "days_to_pay", - "label": "Days to pay", - "type": "int", - "source": "odoo", - "default": false, - "description": "Average days to pay an invoice in full. Blank means no fully paid invoice yet." - }, - { - "key": "top_category", - "label": "Top category", - "type": "text", - "source": "odoo", - "default": false, - "description": "The category this customer spent the most on in the last 12 months." - }, - { - "key": "top_category_pct", - "label": "Top category %", - "type": "pct", - "source": "odoo", - "default": false, - "description": "Share of last-12-months spend that went to the top category." - }, - { - "key": "sku_count", - "label": "SKUs bought", - "type": "int", - "source": "odoo", - "default": false, - "description": "Distinct products bought in the last 12 months." - }, - { - "key": "top_sku", - "label": "Top SKU", - "type": "text", - "source": "odoo", - "default": false, - "description": "The product this customer spent the most on in the last 12 months." - }, - { - "key": "days_since", - "label": "Days since order", - "type": "int", - "source": "odoo", - "default": false, - "description": "Days since the last confirmed order." - }, - { - "key": "typical_gap_days", - "label": "Typical gap days", - "type": "int", - "source": "odoo", - "default": false, - "description": "Days this customer usually goes between orders, from their own history." - }, - { - "key": "notes", - "label": "Notes", - "type": "text", - "source": "overlay", - "default": false, - "description": "Your notes on this customer. Saved in this app only, visible only to you." - } - ], - "_product_comment": "ADDITIVE, wave 15 C-TOPIC. The PRODUCT table's field contract. Kept as a SEPARATE top-level key rather than restructuring `fields` into {customer_data, product_data}: both existing readers (aios_grid._load_fields, aios-web/api/main.py) index doc['fields'] directly, and reshaping that mid-wave would break the embed for a cosmetic gain. The keyed shape can arrive when both readers move in ONE commit; until then this is the product half and `fields` is the customer half.", - "_product_removed_buy_now": "OWNER, 2026-08-03: 'Buy signal' (key buy_now, a select of Buy now / OK) is NO LONGER A PRESET FIELD. It never earned one: it is a formula over two columns that are both still right here, and the platform has a formula field type for exactly that. THE FORMULA, which reproduces the retired column row for row (modules/product_data.validate proves the equivalence, and goes red if it ever stops holding): IF({lead_days} > 0, IF({dos} < {lead_days}, \"Buy now\", \"OK\"), \"\") . Every branch matches the old server rule, including the blanks - the formula engine refuses a comparison against a blank rather than coercing it to 0, so a SKU with no days-of-supply or no lead time comes out empty, which is 'we do not know' and not 'you are fine'. NOTE the column is still COMPUTED in product_data.pool(): it ships nowhere (rows_from_pool projects strictly through this contract, so no Field means no cell on the wire) and exists only as validate()'s oracle. A formula field is PER-USER, so nothing shared may filter on it - the Buy list view filters on dos/lead_days directly (_seed_wave17).", - "product_data": { - "identity": "pid", - "business_key": "code", - "fields": [ - { - "key": "code", - "label": "SKU", - "type": "text", - "source": "odoo", - "pinned": true, - "default": true, - "description": "The SKU code — the product's real business key. `pid` is a stable CRC32 of it because the grid keys on an integer." - }, - { - "key": "product", - "label": "Product", - "type": "text", - "source": "odoo", - "default": true, - "description": "Product name as it appears in Odoo." - }, - { - "key": "category", - "label": "Category", - "type": "select", - "source": "odoo", - "default": true, - "description": "Product category; '(uncategorized)' when Odoo carries none." - }, - { - "key": "supplier", - "label": "Supplier", - "type": "text", - "source": "overlay", - "default": true, - "description": "Who makes it. Editable here and shared with everyone in the workspace; seeded from the inventory mastersheet.", - "shared": true - }, - { - "key": "origin_country", - "label": "Country", - "type": "text", - "source": "overlay", - "default": false, - "description": "Country of origin. Editable here and shared with everyone; seeded from the inventory mastersheet.", - "shared": true - }, - { - "key": "lead_days", - "label": "Lead time (days)", - "type": "int", - "source": "overlay", - "default": true, - "description": "Order-to-arrival days for this supplier. Drives the buy signal. Editable and shared with everyone.", - "shared": true - }, - { - "key": "first_cost", - "label": "First cost", - "type": "currency", - "source": "overlay", - "default": false, - "description": "Quoted unit cost at origin, before freight and duty. Editable and shared with everyone.", - "shared": true - }, - { - "key": "price_fisch", - "label": "Fisch price", - "type": "currency", - "source": "odoo", - "description": "Fisch pricelist price for this SKU. Blank when that list prices it nowhere." - }, - { - "key": "price_royal_1", - "label": "Royal 1 price", - "type": "currency", - "source": "odoo", - "description": "Royal 1 pricelist price for this SKU. Blank when that list prices it nowhere." - }, - { - "key": "price_royal_2", - "label": "Royal 2 price", - "type": "currency", - "source": "odoo", - "description": "Royal 2 pricelist price for this SKU. Blank when that list prices it nowhere." - }, - { - "key": "tier_prices", - "label": "Tier prices", - "type": "json", - "source": "odoo", - "description": "Every live pricelist that prices this SKU today, as [{pricelist, unit_price}] at qty 1. Blank when no list prices it. This is the honest SET; the three Fisch/Royal columns beside it are the DECLARED subset and cannot show a price on a list the contract does not name." - }, - { - "key": "units", - "label": "Units", - "type": "json", - "source": "odoo", - "description": "The units of measure this SKU is really sold in, as [{name, qty}] where qty is in the product's own unit. Blank means it is sold in ONE unit, not that data is missing - only 1,150 of 5,873 active SKUs (19.6%) carry a unit tier." - }, - { - "key": "rev_ytd", - "label": "Revenue YTD", - "type": "currency", - "source": "odoo", - "default": true, - "description": "Year-to-date revenue for this SKU, BU-scoped when the caller is." - }, - { - "key": "rev_ly", - "label": "Revenue LY", - "type": "currency", - "source": "odoo", - "description": "Same period last year — seasonal wholesale compares like for like." - }, - { - "key": "yoy_pct", - "label": "YoY %", - "type": "pct", - "source": "odoo", - "description": "Year-over-year change; null when last year was zero (a ratio to zero is not a number)." - }, - { - "key": "qty_ytd", - "label": "Units YTD", - "type": "int", - "source": "odoo", - "description": "Units sold year to date." - }, - { - "key": "orders_ytd", - "label": "Orders YTD", - "type": "int", - "source": "odoo", - "description": "Distinct orders containing this SKU, year to date." - }, - { - "key": "on_hand", - "label": "On hand", - "type": "int", - "source": "odoo", - "description": "Units in stock. CONSOLIDATED — one physical warehouse, not brand-tagged, so this column is ABSENT for a BU-scoped caller rather than silently company-wide." - }, - { - "key": "unit_cost", - "label": "Unit cost", - "type": "currency", - "source": "odoo", - "description": "Inventory unit cost. Consolidated; absent for a BU-scoped caller." - }, - { - "key": "inv_value", - "label": "Stock value", - "type": "currency", - "source": "odoo", - "description": "On-hand value at cost. Consolidated; absent for a BU-scoped caller." - }, - { - "key": "qty_ltm", - "label": "Units LTM", - "type": "int", - "source": "odoo", - "description": "Units sold in the last twelve months. Consolidated; absent for a BU-scoped caller." - }, - { - "key": "incoming", - "label": "Inbound units", - "type": "int", - "source": "odoo", - "description": "Units already on order and not yet received, from Odoo's own incoming quantity on the product. In the product's STOCK unit of measure, the same unit as On hand, so the two can be added. Days of supply and both cover gap columns all count this as stock (owner, 2026-08-19)." - }, - { - "key": "dos", - "label": "Days of supply", - "type": "int", - "source": "odoo", - "description": "Days of supply at the trailing twelve month rate, counting ON HAND PLUS INBOUND units as stock; null means it never sells through. Same stock figure as the cover gap columns, so the two cannot disagree about how much you have. They still divide by different rates: this one is the trailing average, the cover gap uses the forward 8 month forecast, so a seasonal SKU reads differently in each. Consolidated; absent for a BU-scoped caller." - }, - { - "key": "cover_gap_d", - "label": "Cover gap (days)", - "type": "int", - "source": "odoo", - "default": false, - "description": "Days of cover minus supplier lead time. Negative means it runs out before a reorder lands. It counts INBOUND units as stock, exactly as Days of supply now does. The one thing it does differently: the burn rate is the FORWARD 8 month forecast rather than the trailing twelve month average, so a seasonal SKU reads differently here." - }, - { - "key": "cover_gap_units", - "label": "Cover gap (units)", - "type": "int", - "source": "odoo", - "default": false, - "description": "The recommended reorder quantity: units of forecast demand over the lead time that on-hand plus inbound does not cover. POSITIVE means buy this many; negative is surplus units; blank means we cannot say, because the SKU has no lead time on file or no forecast demand. Rounded AWAY from zero, so a real shortfall never rounds down to nothing." - }, - { - "key": "demand_fwd", - "label": "Forecast units (8 mo)", - "type": "int", - "source": "odoo", - "description": "Units this SKU is expected to sell over the next 8 months, read as the units it actually sold in the same 8 calendar months one year ago. Seasonal on purpose: a flat annual average spreads Valentine's, Mother's Day and Christmas evenly across the year and understates the months buyers actually order for. A SKU too new to appear in that window falls back to its trailing twelve month rate scaled to 8 months. This is the burn rate both cover gap columns divide by, shown so the reorder quantity can be checked." - }, - { - "key": "stock_bucket", - "label": "Stock status", - "type": "select", - "source": "odoo", - "description": "Dead / excess / healthy bucket, from the same days of supply figure beside it, so it counts inbound units too. One exception on purpose: 'Out of stock' still keys on the REAL shelf, because that is a present tense fact somebody can walk into the warehouse and check. A row can honestly read 'Out of stock' with a days of supply beside it when the replenishment is on the water; Inbound units is why. Consolidated; absent for a BU-scoped caller." - }, - { - "key": "discontinued", - "label": "Discontinued", - "type": "select", - "source": "odoo", - "options": [ - "Yes", - "No" - ], - "description": "Whether Odoo carries the Discontinued product tag on this SKU. Every row gets an explicit Yes or No rather than a blank, because a blank reads as an inactive condition in the filter engine and would silently WIDEN any view that filtered on it." - }, - { - "key": "needs_pricing", - "label": "Needs pricing", - "type": "select", - "source": "overlay", - "default": false, - "options": [ - "Yes" - ], - "shared": true, - "description": "Team-maintained. A SKU carries “Yes” when it appears on the NEEDS PRICING tab of the 2027 catalog workbook; no value means it is not on that list. Shared with everyone in the workspace — the whole point of R8 is that the TEAM's work lands in the system, not one importer's private column." - }, - { - "key": "march_pricelist", - "label": "March pricelist", - "type": "select", - "source": "overlay", - "default": false, - "options": [ - "Yes" - ], - "shared": true, - "description": "Team-maintained. A SKU carries “Yes” when it appears on the March Pricelist tab of the 2027 catalog workbook; no value means it is not on that list. Shared with everyone in the workspace — the whole point of R8 is that the TEAM's work lands in the system, not one importer's private column." - }, - { - "key": "price_changes", - "label": "Price changes", - "type": "select", - "source": "overlay", - "default": false, - "options": [ - "Yes" - ], - "shared": true, - "description": "Team-maintained. A SKU carries “Yes” when it appears on the Price Changes tab of the 2027 catalog workbook; no value means it is not on that list. Shared with everyone in the workspace — the whole point of R8 is that the TEAM's work lands in the system, not one importer's private column." - }, - { - "key": "closeouts", - "label": "Closeouts", - "type": "select", - "source": "overlay", - "default": false, - "options": [ - "Yes" - ], - "shared": true, - "description": "Team-maintained. A SKU carries “Yes” when it appears on the Closeouts tab of the 2027 catalog workbook; no value means it is not on that list. Shared with everyone in the workspace — the whole point of R8 is that the TEAM's work lands in the system, not one importer's private column." - }, - { - "key": "notes", - "label": "Notes", - "type": "text", - "source": "overlay", - "default": false, - "shared": true, - "description": "Team-maintained. The 2027 workbook's own non-Odoo columns (Product Description, Packing) folded into one field. Shared with everyone in the workspace." - } - ] - } -} +{ + "_comment": "CANONICAL field contract for the AIOS Airtable-style grid — the SINGLE source of truth. Consumed by platform/aios_grid.py (embed/Space host) and aios-web/api/main.py (standalone API), and regenerated into aios-web/web/public/sample_customers.json. Edit HERE only, then run aios-web/verify_fields_contract.py. source=odoo is READ-ONLY; source=overlay is the editable stratum (notes/tags) outside Odoo. type in {text,status,select,currency,int,date,pct} (select = a fixed-choice READ-ONLY brand attribute; dba is the first, wave 2026-08-02). `description` (wave 5) is the CANONICAL per-field description — every field must carry one, and since wave 7 (owner W8, 2026-07-28) every description is ONE SHORT PLAIN sentence (two only when a fact would otherwise mislead): what the field IS, nothing else — no filter tips, no '(none)' coaching, no rationale; the user's workspace NOTE overrides it in the (i) hover, never in this file. BUILDER FACT (documented here, deliberately NOT in user-facing text): blank text attributes display as '(none)', so `is '(none)'` — not `is empty` — finds the blanks on agent/city/state/country/zip/payment_terms/pricelist/tags. filterable:false = the CONDITION BUILDER does not offer it (still displayed, still sortable); every such field must have a replacement declared in aios-web/verify_fields_contract.py. 2026-07-27 partner attributes: country/zip/payment_terms/pricelist/tags/customer_since all ship default:false. zip is TEXT because a postal code has leading zeros. Odoo's credit_limit (1% populated) and user_id salesperson (2%) are deliberately ABSENT; agent_ids is the salesperson field and AR is where credit exposure comes from. Wave-5 item 8 (2026-07-27): ltm_rev and at_risk are DELETED — LTM's replacement is a creatable Sales measure column (the demo column IS Sales · the last 12 months), at_risk's replacement is a formula field, e.g. MAX(0, {revenue_ly} - {revenue_ytd}). Wave-6 item 8 (2026-07-27, the no-buildable-presets rule): revenue_ytd, revenue_ly, orders_24m, aov and yoy_pct are DELETED — every one is self-buildable, so a frozen pre-set beside the builder was two ways to ask one question. Replacements (recorded in verify_fields_contract.py): creatable measure columns for Sales / Orders / Avg order $ over any period (harness/measure_filter.py ADMITTED carries revenue, orders and the composite aov), and a formula over two measure columns for YoY, e.g. ({sales_ytd} - {sales_ly}) / {sales_ly}. Stale view colIds naming the five self-heal on the next autosave (the established rule).", + "fields": [ + { + "key": "customer", + "label": "Customer", + "type": "text", + "source": "odoo", + "pinned": true, + "default": true, + "description": "The customer's name in Odoo. One row per customer who ordered in the last 24 months." + }, + { + "key": "partner_id", + "label": "Odoo ID", + "type": "int", + "source": "odoo", + "derived": true, + "default": false, + "description": "The Odoo res.partner id — the key every Odoo document joins on. DERIVED: this row's pid IS the partner id, so a stored copy would be a second source." + }, + { + "key": "odoo_status", + "label": "Odoo record", + "type": "status", + "source": "odoo", + "default": false, + "options": [ + "Active", + "Archived" + ], + "description": "Whether this customer still exists in Odoo. Archived means deleted there." + }, + { + "key": "agent", + "label": "Agent", + "type": "text", + "source": "odoo", + "default": true, + "description": "The sales agent who owns this account." + }, + { + "key": "dba", + "label": "DBA", + "type": "select", + "source": "odoo", + "default": false, + "options": [ + "Fisch", + "Royal", + "Both" + ], + "description": "The brand this customer buys from - Fisch, Royal, or both. Amazon-channel orders are not a DBA." + }, + { + "key": "salesperson", + "label": "Salesperson", + "type": "text", + "source": "odoo", + "default": false, + "description": "Who keyed in most of this customer's orders — not the Agent, who owns the account." + }, + { + "key": "street", + "label": "Street", + "type": "text", + "source": "odoo", + "default": false, + "description": "First address line, from res.partner directly - not the geocoder, so a customer the map cannot place still shows its address." + }, + { + "key": "street2", + "label": "Street 2", + "type": "text", + "source": "odoo", + "default": false, + "description": "Second address line (suite, unit, floor) on the customer's Odoo address." + }, + { + "key": "city", + "label": "City", + "type": "text", + "source": "odoo", + "default": true, + "description": "City on the customer's Odoo address." + }, + { + "key": "state", + "label": "State", + "type": "text", + "source": "odoo", + "default": true, + "description": "State or province on the customer's Odoo address." + }, + { + "key": "country", + "label": "Country", + "type": "text", + "source": "odoo", + "default": false, + "description": "Country on the customer's Odoo address." + }, + { + "key": "zip", + "label": "ZIP", + "type": "text", + "source": "odoo", + "default": false, + "description": "Postal code on the customer's Odoo address." + }, + { + "key": "customer_since", + "label": "Customer since", + "type": "date", + "source": "odoo", + "default": false, + "description": "When this customer was first set up in Odoo." + }, + { + "key": "tags", + "label": "Tags", + "type": "text", + "source": "odoo", + "default": false, + "description": "Odoo labels on this customer, comma-separated." + }, + { + "key": "pricelist", + "label": "Price list", + "type": "text", + "source": "odoo", + "default": false, + "description": "The price list this customer buys on." + }, + { + "key": "payment_terms", + "label": "Payment terms", + "type": "text", + "source": "odoo", + "default": false, + "description": "Payment terms on this customer's account — Net 30, for example." + }, + { + "key": "last_order", + "label": "Last order", + "type": "date", + "source": "odoo", + "default": true, + "description": "Date of the most recent confirmed order." + }, + { + "key": "overdue_days", + "label": "Overdue days", + "type": "int", + "source": "odoo", + "default": true, + "description": "How many days late this customer is running against their own usual ordering rhythm." + }, + { + "_note": "filterable:false — DERIVED ANALYTIC: est_missed is min(cycles missed, 3) x AOV, a score we compute rather than an object the business has, so a condition on it would read as a fact about the customer when it is a fact about our arithmetic. It still displays and still sorts. Until wave 6 this flag also covered the frozen-window presets (revenue_ytd / revenue_ly / orders_24m / aov / yoy_pct); those are now DELETED outright under the owner's no-buildable-presets rule — see _comment. est_missed itself STAYS: no creatable measure or formula reproduces the cadence model behind it.", + "key": "est_missed", + "label": "Est. missed $", + "type": "currency", + "source": "odoo", + "default": true, + "agg": "sum", + "filterable": false, + "description": "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed." + }, + { + "_note": "wave 21 R1 — KEY UNCHANGED, LABEL RENAMED. The computation is a DISJOINT split (ar.py credit_exposure): this column is only the not-yet-due residual, its sibling is the past-grace residual, and the two sum to the total. Under the label 'AR open $' the majority-late book read as 'Overdue > Open', which is nonsense in AR vocabulary — 'open' universally means the total. The label now says what the number is; the key stays so saved views and filters keep working.", + "key": "ar_open", + "label": "AR current $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Invoiced money owed but not yet due (a 5-day grace applies before it counts as overdue)." + }, + { + "key": "ar_overdue", + "label": "AR overdue $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Invoiced money past due — same basis as the Collections page." + }, + { + "_note": "wave 21 R1 — the TOTAL, added beside the rename above. AR current $ + AR overdue $, i.e. what most people mean by 'open AR'. Composed from the same ar.credit_exposure rows the siblings use, so it is transitively reconciled by ar.validate()'s residual read_group tie — no second oracle.", + "key": "ar_outstanding", + "label": "AR outstanding $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Total invoiced money owed right now: AR current $ plus AR overdue $." + }, + { + "key": "ar_exposure", + "label": "Credit exposure $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "The most you could be out if they stopped paying today: open, overdue, draft and not-yet-invoiced." + }, + { + "key": "ar_aged_1_30", + "label": "1-30 days $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $." + }, + { + "key": "ar_aged_31_60", + "label": "31-60 days $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $." + }, + { + "key": "ar_aged_61_90", + "label": "61-90 days $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $." + }, + { + "key": "ar_aged_90_plus", + "label": "90+ days $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Overdue by more than 90 days. The four aging buckets sum to AR overdue $." + }, + { + "key": "days_to_pay", + "label": "Days to pay", + "type": "int", + "source": "odoo", + "default": false, + "description": "Average days to pay an invoice in full. Blank means no fully paid invoice yet." + }, + { + "key": "top_category", + "label": "Top category", + "type": "text", + "source": "odoo", + "default": false, + "description": "The category this customer spent the most on in the last 12 months." + }, + { + "key": "top_category_pct", + "label": "Top category %", + "type": "pct", + "source": "odoo", + "default": false, + "description": "Share of last-12-months spend that went to the top category." + }, + { + "key": "sku_count", + "label": "SKUs bought", + "type": "int", + "source": "odoo", + "default": false, + "description": "Distinct products bought in the last 12 months." + }, + { + "key": "top_sku", + "label": "Top SKU", + "type": "text", + "source": "odoo", + "default": false, + "description": "The product this customer spent the most on in the last 12 months." + }, + { + "key": "days_since", + "label": "Days since order", + "type": "int", + "source": "odoo", + "default": false, + "description": "Days since the last confirmed order." + }, + { + "key": "typical_gap_days", + "label": "Typical gap days", + "type": "int", + "source": "odoo", + "default": false, + "description": "Days this customer usually goes between orders, from their own history." + }, + { + "key": "notes", + "label": "Notes", + "type": "text", + "source": "overlay", + "default": false, + "description": "Your notes on this customer. Saved in this app only, visible only to you." + } + ], + "_product_comment": "ADDITIVE, wave 15 C-TOPIC. The PRODUCT table's field contract. Kept as a SEPARATE top-level key rather than restructuring `fields` into {customer_data, product_data}: both existing readers (aios_grid._load_fields, aios-web/api/main.py) index doc['fields'] directly, and reshaping that mid-wave would break the embed for a cosmetic gain. The keyed shape can arrive when both readers move in ONE commit; until then this is the product half and `fields` is the customer half.", + "_product_removed_buy_now": "OWNER, 2026-08-03: 'Buy signal' (key buy_now, a select of Buy now / OK) is NO LONGER A PRESET FIELD. It never earned one: it is a formula over two columns that are both still right here, and the platform has a formula field type for exactly that. THE FORMULA, which reproduces the retired column row for row (modules/product_data.validate proves the equivalence, and goes red if it ever stops holding): IF({lead_days} > 0, IF({dos} < {lead_days}, \"Buy now\", \"OK\"), \"\") . Every branch matches the old server rule, including the blanks - the formula engine refuses a comparison against a blank rather than coercing it to 0, so a SKU with no days-of-supply or no lead time comes out empty, which is 'we do not know' and not 'you are fine'. NOTE the column is still COMPUTED in product_data.pool(): it ships nowhere (rows_from_pool projects strictly through this contract, so no Field means no cell on the wire) and exists only as validate()'s oracle. A formula field is PER-USER, so nothing shared may filter on it - the Buy list view filters on dos/lead_days directly (_seed_wave17).", + "product_data": { + "identity": "pid", + "business_key": "code", + "fields": [ + { + "key": "code", + "label": "SKU", + "type": "text", + "source": "odoo", + "pinned": true, + "default": true, + "description": "The SKU code — the product's real business key. `pid` is a stable CRC32 of it because the grid keys on an integer." + }, + { + "key": "product", + "label": "Product", + "type": "text", + "source": "odoo", + "default": true, + "description": "Product name as it appears in Odoo." + }, + { + "key": "category", + "label": "Category", + "type": "select", + "source": "odoo", + "default": true, + "description": "Product category; '(uncategorized)' when Odoo carries none." + }, + { + "key": "supplier", + "label": "Supplier", + "type": "text", + "source": "overlay", + "default": true, + "description": "Who makes it. Editable here and shared with everyone in the workspace; seeded from the inventory mastersheet.", + "shared": true + }, + { + "key": "origin_country", + "label": "Country", + "type": "text", + "source": "overlay", + "default": false, + "description": "Country of origin. Editable here and shared with everyone; seeded from the inventory mastersheet.", + "shared": true + }, + { + "key": "lead_days", + "label": "Lead time (days)", + "type": "int", + "source": "overlay", + "default": true, + "description": "Order-to-arrival days for this supplier. Drives the buy signal. Editable and shared with everyone.", + "shared": true + }, + { + "key": "first_cost", + "label": "First cost", + "type": "currency", + "source": "overlay", + "default": false, + "description": "Quoted unit cost at origin, before freight and duty. Editable and shared with everyone.", + "shared": true + }, + { + "key": "price_fisch", + "label": "Fisch price", + "type": "currency", + "source": "odoo", + "description": "Fisch pricelist price for this SKU. Blank when that list prices it nowhere." + }, + { + "key": "price_royal_1", + "label": "Royal 1 price", + "type": "currency", + "source": "odoo", + "description": "Royal 1 pricelist price for this SKU. Blank when that list prices it nowhere." + }, + { + "key": "price_royal_2", + "label": "Royal 2 price", + "type": "currency", + "source": "odoo", + "description": "Royal 2 pricelist price for this SKU. Blank when that list prices it nowhere." + }, + { + "key": "tier_prices", + "label": "Tier prices", + "type": "json", + "source": "odoo", + "description": "Every live pricelist that prices this SKU today, as [{pricelist, unit_price}] at qty 1. Blank when no list prices it. This is the honest SET; the three Fisch/Royal columns beside it are the DECLARED subset and cannot show a price on a list the contract does not name." + }, + { + "key": "units", + "label": "Units", + "type": "json", + "source": "odoo", + "description": "The units of measure this SKU is really sold in, as [{name, qty}] where qty is in the product's own unit. Blank means it is sold in ONE unit, not that data is missing - only 1,150 of 5,873 active SKUs (19.6%) carry a unit tier." + }, + { + "key": "rev_ytd", + "label": "Revenue YTD", + "type": "currency", + "source": "odoo", + "default": true, + "description": "Year-to-date revenue for this SKU, BU-scoped when the caller is." + }, + { + "key": "rev_ly", + "label": "Revenue LY", + "type": "currency", + "source": "odoo", + "description": "Same period last year — seasonal wholesale compares like for like." + }, + { + "key": "yoy_pct", + "label": "YoY %", + "type": "pct", + "source": "odoo", + "description": "Year-over-year change; null when last year was zero (a ratio to zero is not a number)." + }, + { + "key": "qty_ytd", + "label": "Units YTD", + "type": "int", + "source": "odoo", + "description": "Units sold year to date." + }, + { + "key": "orders_ytd", + "label": "Orders YTD", + "type": "int", + "source": "odoo", + "description": "Distinct orders containing this SKU, year to date." + }, + { + "key": "on_hand", + "label": "On hand", + "type": "int", + "source": "odoo", + "description": "Units in stock. CONSOLIDATED — one physical warehouse, not brand-tagged, so this column is ABSENT for a BU-scoped caller rather than silently company-wide." + }, + { + "key": "unit_cost", + "label": "Unit cost", + "type": "currency", + "source": "odoo", + "description": "Inventory unit cost. Consolidated; absent for a BU-scoped caller." + }, + { + "key": "inv_value", + "label": "Stock value", + "type": "currency", + "source": "odoo", + "description": "On-hand value at cost. Consolidated; absent for a BU-scoped caller." + }, + { + "key": "qty_ltm", + "label": "Units LTM", + "type": "int", + "source": "odoo", + "description": "Units sold in the last twelve months. Consolidated; absent for a BU-scoped caller." + }, + { + "key": "incoming", + "label": "Inbound units", + "type": "int", + "source": "odoo", + "description": "Units already on order and not yet received, from Odoo's own incoming quantity on the product. In the product's STOCK unit of measure, the same unit as On hand, so the two can be added. Days of supply and both cover gap columns all count this as stock (owner, 2026-08-19)." + }, + { + "key": "dos", + "label": "Days of supply", + "type": "int", + "source": "odoo", + "description": "Days of supply at the trailing twelve month rate, counting ON HAND PLUS INBOUND units as stock; null means it never sells through. Same stock figure as the cover gap columns, so the two cannot disagree about how much you have. They still divide by different rates: this one is the trailing average, the cover gap uses the forward 8 month forecast, so a seasonal SKU reads differently in each. Consolidated; absent for a BU-scoped caller." + }, + { + "key": "cover_gap_d", + "label": "Cover gap (days)", + "type": "int", + "source": "odoo", + "default": false, + "description": "Days of cover minus supplier lead time. Negative means it runs out before a reorder lands. It counts INBOUND units as stock, exactly as Days of supply now does. The one thing it does differently: the burn rate is the FORWARD 8 month forecast rather than the trailing twelve month average, so a seasonal SKU reads differently here." + }, + { + "key": "cover_gap_units", + "label": "Cover gap (units)", + "type": "int", + "source": "odoo", + "default": false, + "description": "The recommended reorder quantity: units of forecast demand over the lead time that on-hand plus inbound does not cover. POSITIVE means buy this many; negative is surplus units; blank means we cannot say, because the SKU has no lead time on file or no forecast demand. Rounded AWAY from zero, so a real shortfall never rounds down to nothing." + }, + { + "key": "demand_fwd", + "label": "Forecast units (8 mo)", + "type": "int", + "source": "odoo", + "description": "Units this SKU is expected to sell over the next 8 months, read as the units it actually sold in the same 8 calendar months one year ago. Seasonal on purpose: a flat annual average spreads Valentine's, Mother's Day and Christmas evenly across the year and understates the months buyers actually order for. A SKU too new to appear in that window falls back to its trailing twelve month rate scaled to 8 months. This is the burn rate both cover gap columns divide by, shown so the reorder quantity can be checked." + }, + { + "key": "stock_bucket", + "label": "Stock status", + "type": "select", + "source": "odoo", + "description": "Dead / excess / healthy bucket, from the same days of supply figure beside it, so it counts inbound units too. One exception on purpose: 'Out of stock' still keys on the REAL shelf, because that is a present tense fact somebody can walk into the warehouse and check. A row can honestly read 'Out of stock' with a days of supply beside it when the replenishment is on the water; Inbound units is why. Consolidated; absent for a BU-scoped caller." + }, + { + "key": "discontinued", + "label": "Discontinued", + "type": "select", + "source": "odoo", + "options": [ + "Yes", + "No" + ], + "description": "Whether Odoo carries the Discontinued product tag on this SKU. Every row gets an explicit Yes or No rather than a blank, because a blank reads as an inactive condition in the filter engine and would silently WIDEN any view that filtered on it." + }, + { + "key": "needs_pricing", + "label": "Needs pricing", + "type": "select", + "source": "overlay", + "default": false, + "options": [ + "Yes" + ], + "shared": true, + "description": "Team-maintained. A SKU carries “Yes” when it appears on the NEEDS PRICING tab of the 2027 catalog workbook; no value means it is not on that list. Shared with everyone in the workspace — the whole point of R8 is that the TEAM's work lands in the system, not one importer's private column." + }, + { + "key": "march_pricelist", + "label": "March pricelist", + "type": "select", + "source": "overlay", + "default": false, + "options": [ + "Yes" + ], + "shared": true, + "description": "Team-maintained. A SKU carries “Yes” when it appears on the March Pricelist tab of the 2027 catalog workbook; no value means it is not on that list. Shared with everyone in the workspace — the whole point of R8 is that the TEAM's work lands in the system, not one importer's private column." + }, + { + "key": "price_changes", + "label": "Price changes", + "type": "select", + "source": "overlay", + "default": false, + "options": [ + "Yes" + ], + "shared": true, + "description": "Team-maintained. A SKU carries “Yes” when it appears on the Price Changes tab of the 2027 catalog workbook; no value means it is not on that list. Shared with everyone in the workspace — the whole point of R8 is that the TEAM's work lands in the system, not one importer's private column." + }, + { + "key": "closeouts", + "label": "Closeouts", + "type": "select", + "source": "overlay", + "default": false, + "options": [ + "Yes" + ], + "shared": true, + "description": "Team-maintained. A SKU carries “Yes” when it appears on the Closeouts tab of the 2027 catalog workbook; no value means it is not on that list. Shared with everyone in the workspace — the whole point of R8 is that the TEAM's work lands in the system, not one importer's private column." + }, + { + "key": "notes", + "label": "Notes", + "type": "text", + "source": "overlay", + "default": false, + "shared": true, + "description": "Team-maintained. The 2027 workbook's own non-Odoo columns (Product Description, Packing) folded into one field. Shared with everyone in the workspace." + } + ] + } +} diff --git a/platform/core/perm_scope.py b/platform/core/perm_scope.py index 6da0fead55094307735e999c7cb13564c377d5a4..3b707578375d940283cecbe032f6274a9b178414 100644 --- a/platform/core/perm_scope.py +++ b/platform/core/perm_scope.py @@ -1,999 +1,999 @@ -"""core/perm_scope.py — the permission WALL for table modules (wave 15, C-PERM). - -ONE place answers the three questions a restricted account raises on a grid surface: - - may_access(user, module) may they open it at all? - visible_fields(fields, u, mod) which COLUMNS may they receive? - apply_row_scope(rows, u, mod, …) which ROWS may they receive? - -plus one that exists only because of how the pool is built: - - derive_pool_scope(user, module) which (team_id, agent) must the pool be BUILT with? - -Both hosts call these — `aios-web/api` (`grid_assembly`) and `app.py` (`_table_grid`) — because -a wall that exists on one runtime and not the other is not a wall. `core/perms.py` stays what it -is (module GRANTS + the legacy BU derivation); this module is the row/field/pushdown layer that -sits on top, and it is deliberately a separate file so the legacy readers can keep their -semantics untouched while this one fails closed. - -──────────────────────────────────────────────────────────────────────────────────────────── -THE RECORD - - user['perms'] = {'': {'access': bool, - 'filter': {'conj'?: 'and'|'or', 'nodes': [...]} | None, - 'hiddenFields': ['', ...]}} - user['perms_v'] = 1 stamped by the migration and by every write - -`perms_v` is the EXPLICIT-RESOLUTION marker, and it is here because of `permissioning.md` -Part II gap #5: "`allowed_modules() is None` is fail-open by default … make 'resolved: -unrestricted' an explicit value so absence/uncertainty DENIES." The same class already shipped -twice in this codebase (`modules: []` and `bus: []` both read as UNRESTRICTED — -`routes_admin.py`'s own docstring documents both). So: - - * `perms_v` ABSENT → the record is UN-MIGRATED, and the LEGACY wall applies unchanged - (`core.perms` module grants + the `bus`/`agent` query scope). That is not fail-open: it is - today's real wall, and it bounds the rollout window to "until the migration runs". - * `perms_v == 1` and the module has NO entry → **DENY**. Absence now means what it says. - * `role == 'admin'` bypasses all of it — which is also what keeps BREAK-GLASS alive. - `deps._user_for` hands back a hardcoded master dict on a store outage - (`{'username':'admin','role':'admin','bus':'all','modules':'all'}`) that will never carry a - perms block; without this clause an explicit-marker scheme locks the owner out of their own - product at exactly the moment the store is broken. -""" -import re - -import core.perms as perms - -#: `aios_grid._FORMULA_REF`'s pattern, restated rather than imported: this module is imported by -#: the API's request path and `aios_grid` pulls in the whole grid stack. Same regex, one line, -#: and `verify_api` asserts the two agree so it cannot drift into a different grammar. -_FORMULA_REF = re.compile(r"\{([^{}]*)\}") - -PERMS_VERSION = 1 - - -def _rec(user): - return user if isinstance(user, dict) else {} - - -def is_migrated(user): - """True once this record carries an explicit resolution. See the module docstring.""" - return int(_rec(user).get('perms_v') or 0) >= PERMS_VERSION - - -def entry(user, module): - """This user's declared permissions for `module`, or None if nothing is declared. - - None is AMBIGUOUS on purpose and every caller must resolve it against `is_migrated`: - on a migrated record it means DENY, on a legacy one it means "ask the old wall". - """ - p = _rec(user).get('perms') - if not isinstance(p, dict): - return None - e = p.get(module) - return e if isinstance(e, dict) else None - - -def may_access(user, module): - """May this account open `module` at all? Fail-closed on a migrated record.""" - if perms.is_admin(user): - return True - e = entry(user, module) - if e is not None: - return bool(e.get('access', True)) - if is_migrated(user): - # Migrated and undeclared = denied. This is the whole point of the marker. - return False - return perms.may_open(user, module) # legacy record: the old grant wall - - -def may_metrics(user, module): - """May this account build and receive METRIC columns — lookback measures — on `module`? - - ⭐⭐ W38-T19 — A CAPABILITY, NOT A SECOND SPELLING OF ACCESS, and the distinction is the - ticket. A rollup aggregates the CHILDREN a row is linked to; a Metric answers *"this number, - over this window"* against a governed topic with no relation at all (CLAUDE.md standing rule - 9). So it reads the book behind the rows rather than the rows: an account can be exactly the - right person to see a customer list and the wrong one to mint 12-month revenue over it. Two - decisions, two controls. - - ⛔ ABSENCE GRANTS, AND THE ASYMMETRY WITH `may_access` IS DELIBERATE RATHER THAN AN - OVERSIGHT TO TIDY. `may_access` reads migrated-and-undeclared as DENY, which is right there - because every save writes an entry for every governed database — absence means an - administrator decided. No `metrics` KEY was STORABLE before this ticket, so every migrated - record in every tenant carries none, and reading that absence as DENY would revoke Metrics - for everybody on the day this shipped with nobody having decided anything. That is the exact - failure `routes_admin.get_perms` records twice already (the `ut_*` default and the surface - default), one field further down the same entry. **Only an explicit `metrics: false` - refuses.** - - ⚠ IT DOES NOT RE-ASK ACCESS. Every caller is already behind the access wall (`module_gate`, - `may_read`, `session.require`), and folding admission in here would give a denial two - possible causes with one answer — the shape that makes a permission bug take an afternoon. - """ - if perms.is_admin(user): - return True - e = entry(user, module) - return not (isinstance(e, dict) and e.get('metrics') is False) - - -def nav_may_open(user, key, st=None): - """May this account SEE `key` — in the nav, and at the route gate? The ADMISSION question. - - ⛔⛔ WHY THIS EXISTS: THE EDITOR'S DECISION REACHED THE READ DOOR AND NOTHING ELSE. Measured - on live 2026-08-18, on a real account. An administrator ticked *Odoo products* for Naomi in - Manage user and saved; the record stored `perms.product_data.access = True` and - `may_access()` agreed. **The database never appeared.** `perms.nav_pages` and - `deps.Session.require` both ask `perms.may_open` — the LEGACY `user['modules']` array — which - still read `['sales', 'customers', 'products']`, and `products` is the key of the ARCHIVED - *SKU* module, not of `product_data`. So `nav_pages` answered `['customer_data']` and the - route would have 403'd her even by URL. Two permission systems, the editor writing one and - every DOOR reading the other ([[two-permission-systems-one-armed]]). - - ⭐ THE ASYMMETRY IS THE WHOLE DESIGN, and it is not the same rule twice: - - 1. an admin sees everything (break-glass, as everywhere else); - 2. an EXPLICIT `access: false` DENIES, on any key — an administrator unticking a box must - take the row off the nav, and before this it did not; - 3. a `ut_*` key with no explicit deny defers to `user_tables.may_open` — creator, admin or - a `core.shares` grant — and **an `access: true` entry may NEVER widen past it**. The - editor must not become a way to hand somebody another user's private table; - 4. a REGISTRY TOPIC with an explicit entry takes that entry, grant included. Here the - editor IS the authority: `_clean_perms` writes an entry for every governed topic on - every save, so an entry means an administrator decided; - 5. anything else — a module this editor does not govern — falls through to - `perms.may_open`, UNCHANGED. - - ⛔ LEG 5 IS LOAD-BEARING AND IT IS WHY `may_access` COULD NOT SIMPLY BE CALLED HERE. - `may_access` reads migrated-and-undeclared as DENY, which is correct for a governed topic and - catastrophic for the nav: Naomi's block declares the ten governed keys and nothing else, so - `sales` would have gone from visible to denied — an outage dressed as a permission fix. - """ - if perms.is_admin(user): - return True - e = entry(user, key) - if e is not None and not bool(e.get('access', True)): - return False - if str(key or '').startswith(_ut().KEY_PREFIX): - return bool(_ut().may_open(key, (user or {}).get('username'), False, st=st)) - if e is not None: - return bool(e.get('access', True)) - return perms.may_open(user, key) - - -def assistant_entry(user, module): - """The explicit database grant an Assistant snapshot may rely on, else ``None``. - - The interactive application retains an admin break-glass path and a temporary legacy-grant - compatibility path. Neither is an answer at the Assistant's app-stored data boundary: - that reader must be able to name the migrated grant that admitted a database. In particular, - a store-outage admin identity with no ``perms`` document is not an unresolved permission that - may be widened into data access. - """ - e = entry(user, module) - if (not is_migrated(user) or not isinstance(e, dict) - or not bool(e.get('access', True)) or not may_access(user, module)): - return None - return e - - -# ── FIELDS ─────────────────────────────────────────────────────────────────────────────────── -#: ⭐⭐ W38-T16 — THE MARKER THAT SAYS "THIS COLUMN IS GOVERNED BY A GRANT", stamped once by the -#: door that creates a shared column (`routes_tables.patch_shared_cell`) and never flipped by any -#: door afterwards. It is the `perms_v` idiom one object down, and it is here for the reason that -#: marker exists: **absence must not be read as a decision.** -#: -#: ⛔⛔ WHY A MARKER AND NOT "DOES A GRANT RECORD EXIST". Every tenant-wide column shipped before -#: this ticket carries no `field` grant record, because none was STORABLE — `shares.KINDS` had -#: three members. Reading that absence as "granted to nobody" would blank every existing shared -#: column for every account the moment this arms, which is not a permission fix, it is an outage -#: (`may_read` leg 3 carries the same argument for `ut_*` keys, in the same words). -#: -#: ⛔⛔ AND IT IS WHAT MAKES THE WALL FAIL **CLOSED**. The polarity here is the opposite of every -#: other grant check in this repo: a field share is a GRANT, so "no grant" has to mean HIDDEN or -#: the wall does nothing — but "no grant record" also describes a legacy column and a store that -#: could not be read. The marker separates the three: `granted` on the column means an explicit -#: decision was taken, so an unreadable registry hides it; no marker means legacy, so nothing -#: changes. Without it, one unreadable read of `object_shares` would publish every governed -#: column to the whole tenant, silently ([[a-guard-for-the-dangerous-case]]). -#: -#: ⚠ IT IS NOT `shared_overlay`'s `"shared": True` AND MUST NOT BE CONFUSED WITH IT. That flag is -#: written and never read (D-414) — `is_shared` is a dict-membership test — so it is not evidence -#: of anything. This one is read HERE, on every assembly, and the only writer is the create door. -FIELD_GRANT_MARK = 'granted' - - -def granted_field_keys(fields): - """Every column in `fields` that declares itself GOVERNED by a `shares` grant. - - ⭐ THE CHEAP HALF OF THE WALL, AND IT IS WHY THE WALL COSTS NOTHING FOR ALMOST EVERYBODY. It - is a scan of dicts already in memory, so a database with no governed column reaches no store - at all and `hidden_keys` behaves exactly as it did before this ticket. The registry is only - opened once this answers non-empty. - """ - return {str(f['key']) for f in (fields or ()) - if isinstance(f, dict) and f.get('key') - and f.get(FIELD_GRANT_MARK) is True} - - -def field_grant_hidden(user, table_key, fields, st=None): - """The governed columns of `table_key` this principal holds NO grant on — C1's per-FIELD wall. - - ⭐⭐ W38-T16 / R7 / R8 — THE THIRD WALL, AND IT COMPOSES ALONGSIDE THE OTHER TWO RATHER THAN - INSIDE THEM. `may_open` answers *IF* you reach a database; `may_read`'s stored `access: false` - overlay may REVOKE one; this answers *WHICH COLUMNS* of it you receive. It is deliberately not - threaded through that overlay: the overlay is **deny-only** by its own docstring (*"it may - revoke a database the wall below would admit, and it may never grant one that wall refuses"*) - and a field share is a GRANT — the widening direction. Merging them would give the codebase a - second idea of who grants what, which is the failure `may_open`'s own note is the record of. - It composes the way `may_open`'s grant leg does: additively, last, fail-closed. - - ⛔ AN ADMIN IS NOT WALLED (break-glass, as everywhere else) and neither is the column's OWNER — - `shares.role_for` answers `'owner'` for the creator, so a person cannot lose their own column - by forgetting to share it with themselves. - - ⚠ `st` IS THE TENANT HANDLE AND ITS ABSENCE IS SAFE HERE, unlike everywhere else. A caller - that cannot lend one reads the module-default bucket; on any tenant but #0 that finds no - grant, and no grant on a MARKED column means HIDDEN. So a door that has not learned to thread - `st` under-shares rather than over-shares, and the symptom is a grantee who cannot see their - column — visible, reportable, and the opposite of a leak. - """ - if perms.is_admin(user): - return set() - marked = granted_field_keys(fields) - if not marked: - return set() - uname = str((user or {}).get('username') or '').strip().lower() - try: - import core.shares as shares - except Exception: # noqa: BLE001 - return set(marked) - if not uname: - # No principal, and a marked column is an explicit decision: nobody is not somebody. - return set(marked) - # ⭐ THE CREATOR IS READ OFF THE COLUMN, NOT OUT OF THE REGISTRY, AND THAT IS NOT A SECOND - # AUTHORITY. `createdBy` is ALREADY what decides who may DELETE a shared column (R8 / D-172, - # `routes_tables.delete_shared_field`) and who may CLAIM it (`routes_shares._owns_object`); - # asking the same field here keeps one answer to "whose column is this" across all three. - # ⛔ IT IS ALSO THE ONLY THING THAT SURVIVES A CLAIM THAT NEVER LANDED. The create door writes - # the definition first and the grant record second, on purpose — so the window where a column - # is marked and unclaimed exists, and without this line its own author would be walled out of - # the column they just made, permanently and with no way to fix it but an admin. - mine = {str(f['key']) for f in (fields or ()) - if isinstance(f, dict) and f.get('key') and str(f['key']) in marked - and str(f.get('createdBy') or '').strip().lower() == uname} - hide = set() - for key in marked - mine: - try: - oid = shares.field_oid(table_key, key) - role = shares.role_for('field', oid, uname, is_admin=False, st=st) - except Exception: # noqa: BLE001 - role = None - if role is None: - hide.add(key) - return hide - - -def hidden_keys(user, module, fields, st=None): - """The TRANSITIVE closure of hidden field keys (C-PERM amendment 5). - - ⛔ WHY A CLOSURE AND NOT A SET DIFFERENCE. A formula field is computed in the BROWSER - (`formulaEngine.ts`, injected by `computedRows`) from `{ref}`s into other columns, and a - measure column's value arrives precomputed in `derived`. So hiding field X has exactly three - possible outcomes and only one of them is coherent: - - strip X, keep formulas → every formula over X computes blank or wrong, silently - keep X's value for them → X has leaked, wearing a formula's name - strip X AND its dependents→ the only honest answer - - So a hidden field drags every formula that references it — and every formula that references - THAT formula, hence the fixpoint loop — out of the payload with it. - - ⚠ This runs on every assembly, so it is a fixpoint over a handful of custom fields, not a - graph library. `MAX_PASSES` bounds a reference cycle the client would refuse to evaluate - anyway; without it a self-referential pair would spin here. - - ⭐⭐ W36-T21 — AND A ROLLUP IS THE SAME LEAK ONE MECHANISM OVER, which matters now that this - closure runs on the `ut_*` databases rather than only on the two registry topics. A rollup - names a LINK COLUMN OF THIS TABLE (`rollup.link`) and aggregates a field on the table that - link points at — so `ut_odoo_customers.ar_outstanding` is *"sum `residual` over the invoices - this row links to"*. Hide `invoices` and keep `ar_outstanding` and the reader still learns - what the hidden link contains, in aggregate; the three outcomes are exactly the three the - formula argument above enumerates, and only "strip both" is coherent. Verified against the - real declarations (`odoo_relational.customer_fields`) rather than assumed: every rollup there - is either `{'link': , 'field': }` - or a `source` topic aggregate, so `rollup.link` is the ONE same-table reference a rollup makes - and `rollup.field` is deliberately not treated as one — it names another database's column, - which has its own wall. - """ - if perms.is_admin(user): - return frozenset() - # ⭐⭐ W38-T16 — TWO SOURCES OF HIDING, ONE CLOSURE, AND THAT UNION IS THE WHOLE INTEGRATION. - # - # The administrator's `hiddenFields` and a field's own grant answer different questions and - # both end in the same place: a key this reader may not receive. Seeded together HERE, before - # the fixpoint, so the transitive argument above covers the new source unchanged — a formula - # (or a rollup) over a column this reader was not granted comes out with it, or the value - # leaks wearing the derived column's name. - # - # ⛔ AND THIS FUNCTION IS THE INSERTION POINT RATHER THAN `_scoped`, WHICH IS WHAT THE TICKET - # ASSUMED. `_scoped` is C1's evaluator and reaches C1's two doors; **the product reads through - # neither of them.** Every grid door calls THIS: `routes_customers:196`, `routes_products:345`, - # `routes_odoo_tables:920/1119`, `routes_tables._ut_hidden/_ut_field_wall`, `routes_slack:190`, - # `routes_grid:57/69/787` — and `grid_events` walls WRITES off the same set through - # `EventCtx.hidden_keys`. One evaluator, every door, both wires, read and write. - hidden = set(field_grant_hidden(user, module, fields, st=st)) - e = entry(user, module) - if e: - hidden |= {str(k) for k in (e.get('hiddenFields') or ()) if k} - if not hidden: - return frozenset() - - refs = {} - for f in fields or (): - if not isinstance(f, dict) or not f.get('key'): - continue - expr = f.get('formula') - if isinstance(expr, str) and expr: - refs[f['key']] = {m.strip() for m in _FORMULA_REF.findall(expr) if m.strip()} - link = (f.get('rollup') or {}).get('link') if isinstance(f.get('rollup'), dict) else None - if isinstance(link, str) and link.strip(): - refs.setdefault(f['key'], set()).add(link.strip()) - - MAX_PASSES = 12 - for _ in range(MAX_PASSES): - grew = False - for key, deps in refs.items(): - if key not in hidden and deps & hidden: - hidden.add(key) - grew = True - if not grew: - break - return frozenset(hidden) - - -def visible_fields(fields, user, module, st=None): - """`fields` minus the hidden closure. Order preserved — the column order is the user's. - - ⚠ W38-T16 — `st` MUST MATCH WHAT ITS CALLER PASSED TO `hidden_keys`, and that is not a style - note. This recomputes the closure, and since the field-grant leg under-hides without a tenant - handle, a caller that lends one to `hidden_keys` and not to this would narrow the FIELD LIST - by MORE than it stripped from the ROWS — the value left sitting in the payload under a column - nobody can see, which is exactly the half-wall `strip_row`'s note exists to forbid. - """ - hide = hidden_keys(user, module, fields, st=st) - if not hide: - return list(fields or ()) - return [f for f in (fields or ()) - if not (isinstance(f, dict) and f.get('key') in hide)] - - -def assistant_visible_fields(fields, user, module): - """Visible closure under one explicit Assistant grant, including for an admin caller.""" - e = assistant_entry(user, module) - if e is None: - return [] - # `hidden_keys` deliberately preserves the interactive admin break-glass behaviour. The - # Assistant uses its explicit grant instead, but shares the same transitive formula closure. - hidden = {str(k) for k in (e.get('hiddenFields') or ()) if k} - if not hidden: - return list(fields or ()) - - refs = {} - for field in fields or (): - if not isinstance(field, dict) or not field.get('key'): - continue - expr = field.get('formula') - if isinstance(expr, str) and expr: - refs[field['key']] = {match.strip() for match in _FORMULA_REF.findall(expr) - if match.strip()} - for _ in range(12): - grew = False - for key, deps in refs.items(): - if key not in hidden and deps & hidden: - hidden.add(key) - grew = True - if not grew: - break - return [field for field in (fields or ()) - if not (isinstance(field, dict) and field.get('key') in hidden)] - - -def strip_row(row, hide): - """Drop hidden keys from ONE assembled row. Cheap enough to run per row, and it must run - per row: the field list and the row payload are two different wires, and stripping only the - first would leave the value sitting in the second where anyone can read it.""" - if not hide or not isinstance(row, dict): - return row - return {k: v for k, v in row.items() if k not in hide} - - -# ── ROWS ───────────────────────────────────────────────────────────────────────────────────── -def apply_row_scope(rows, user, module, fields, ctx=None): - """The rows this account may receive: `filter_eval.permits` over the permanent filter. - - `permits`, never `matches` — an unanswerable permanent filter DENIES rather than being - ignored. See `harness/filter_eval`'s docstring for the field-rename walkthrough that makes - the difference a leak rather than a preference. - """ - if perms.is_admin(user): - return list(rows or ()) - e = entry(user, module) - tree = (e or {}).get('filter') - if not tree: - return list(rows or ()) - from harness import filter_eval as fe - return [r for r in (rows or ()) if fe.permits(tree, r, fields, ctx)] - - -def assistant_apply_row_scope(rows, user, module, fields, ctx=None): - """Apply an explicit Assistant grant's permanent filter without an admin bypass.""" - e = assistant_entry(user, module) - if e is None: - return [] - tree = e.get('filter') - if not tree: - return list(rows or ()) - from harness import filter_eval as fe - return [row for row in (rows or ()) if fe.permits(tree, row, fields, ctx)] - - -def validate_assistant_filter(tree, fields): - """Return a strict, detached Assistant filter tree or raise ``ValueError``. - - The display filter cleaner is intentionally permissive: it drops a stale column or leaves an - inactive condition alone so an old saved view can still open. An Assistant data reader cannot - inherit that behaviour — dropping a predicate turns a request for a subset into a wider read. - This validator therefore admits only visible field operands that the existing evaluator can - answer from one stored row. Cohort, measure and rank conditions need separate materialised - set resolvers and are refused here rather than guessed. - """ - if tree is None: - return None - from harness import filter_eval as fe - - by_key = {str(field.get('key')): field for field in (fields or ()) - if isinstance(field, dict) and field.get('key')} - if not by_key: - raise ValueError('assistant filter has no visible field contract') - - def _leaf(raw): - if not isinstance(raw, dict): - raise ValueError('assistant filter leaf must be an object') - allowed = {'id', 'colId', 'op', 'value', 'value2', 'rhs'} - if set(raw) - allowed: - raise ValueError('assistant filter carries an unsupported operand') - col = raw.get('colId') - op = raw.get('op') - if not isinstance(col, str) or col not in by_key: - raise ValueError('assistant filter names an unknown or hidden field') - if (not isinstance(op, str) or op not in fe.FILTER_OPS or op in fe.RANK_OPS - or op in {'between', 'within'} or col == fe.COHORT_FIELD): - raise ValueError('assistant filter uses an unsupported operator') - rhs = raw.get('rhs') - if rhs is not None: - if (not isinstance(rhs, dict) or rhs.get('kind') != 'field' - or set(rhs) - {'kind', 'colId'} - or not isinstance(rhs.get('colId'), str) - or rhs['colId'] not in by_key): - raise ValueError('assistant filter names an unknown or hidden right-hand field') - out = {name: raw[name] for name in ('id', 'colId', 'op', 'value', 'value2') - if name in raw} - if rhs is not None: - out['rhs'] = {'kind': rhs.get('kind'), 'colId': rhs['colId']} - if not fe.is_rule_active(out, by_key): - raise ValueError('assistant filter is inactive or unanswerable') - return out - - def _node(raw): - if not isinstance(raw, dict): - raise ValueError('assistant filter node must be an object') - if 'children' not in raw: - return _leaf(raw) - if set(raw) - {'conj', 'children'}: - raise ValueError('assistant filter group carries an unsupported operand') - children = raw.get('children') - if raw.get('conj') not in ('and', 'or') or not isinstance(children, list) or not children: - raise ValueError('assistant filter group must be a non-empty and/or group') - return {'conj': raw['conj'], 'children': [_node(child) for child in children]} - - if isinstance(tree, list): - if not tree: - raise ValueError('assistant filter list must not be empty') - return {'conj': 'and', 'nodes': [_node(node) for node in tree]} - if not isinstance(tree, dict) or set(tree) - {'conj', 'nodes'}: - raise ValueError('assistant filters must be a tree with nodes') - nodes = tree.get('nodes') - if tree.get('conj') not in ('and', 'or') or not isinstance(nodes, list) or not nodes: - raise ValueError('assistant filter tree must be a non-empty and/or tree') - return {'conj': tree['conj'], 'nodes': [_node(node) for node in nodes]} - - -# ── THE PUSHDOWN ───────────────────────────────────────────────────────────────────────────── -#: `dba` value sets that pin a BU, and the Odoo team they pin. `_dba_attrs` emits exactly -#: {'Fisch','Royal','Both'} (blank for "no brand attributable in 24 months"), so a permission -#: filter admitting Fisch admits {Fisch, Both} — a Both customer IS a Fisch customer. -_DBA_TEAM = ((frozenset({'fisch', 'both'}), 5), (frozenset({'royal', 'both'}), 6)) - -#: Only equality pushes down. `FILTER_OPS` is single-valued (there is no column-level "is any -#: of" — that vocabulary belongs to cohort leaves and is disjoint), so a multi-value BU -#: condition arrives as an OR GROUP of `eq` leaves, handled by `_group_dba_team` below. -_PUSHDOWN_OPS = frozenset({'eq'}) - -#: ⭐ THE MODULES WHOSE FIELD VOCABULARY CAN EXPRESS A BUSINESS UNIT — i.e. whose canonical -#: contract carries a `dba` column. R1's "BU access is just a permanent filter" holds only on -#: these; on every other topic there is no column to write the condition against, so a filter -#: literally cannot say it and the record's `bus` remains the only place the fact lives. -#: -#: `product_data` is the counter-example that made this constant necessary: 19 fields, no brand -#: column, and a product's BU is a property of WHOSE ORDERS built its revenue rather than of the -#: SKU. Without the fallback below, a Fisch-only account read the product catalogue with -#: Fisch+Royal money on every row — the amendment-3 defect, arriving through the door amendment 3 -#: was written to close. -#: -#: ⚠ A LIST THAT MIRRORS A JSON CONTRACT DRIFTS UNLESS SOMETHING CHECKS IT. `verify_api` asserts -#: membership here matches "this topic's contract has a `dba` field" for every governed module, so -#: a topic that grows or loses a brand column cannot silently keep the wrong rule. Named in this -#: module rather than derived from `aios_grid` on purpose: `perm_scope` is on the API's request -#: path and importing the grid stack to answer a two-element question is a cost per request. -BU_FILTERABLE_MODULES = frozenset({'customer_data'}) - - -def derive_pool_scope(user, module): - """`(team_id, agent)` the POOL must be BUILT with, derived from the permanent filter. - - ⛔ THIS EXISTS BECAUSE `team_id` SHAPES VALUES, NOT ROW MEMBERSHIP (C-PERM amendment 3). - `modules/customer_data._pool_build` passes `team_id` into `cust._cust_rev` three times (YTD, - LY, LTM) and into `cust._cadence_bulk`, so it decides what `rev`, `ly`, `ltm`, `aov`, - `est_missed` and the derived `status` MEAN. Enforce a BU purely as a post-filter and a - Fisch-only user keeps a correct-looking row LIST while every number on it silently becomes - Fisch+Royal — worst for `dba = Both` customers, who are exactly the ones a BU filter admits. - A pid-level reconciliation cannot see that; the values one can, and does. - - So the query-level pushdown SURVIVES — but as a DERIVATION OF the permanent filter rather - than a second wall beside it, which is what keeps R1's "BU access is just a filter" true at - the level the owner asked for it (one declaration, one UI, one engine). - - Pure, and recomputed per request rather than stored: a stored derivation drifts from the - filter it came from, and then two things disagree about what an account may see. - - Reads TOP-LEVEL AND-conjunction leaves ONLY. A leaf under `or` guarantees no narrowing — - `dba is Fisch OR revenue > 10` must not pin the pool to Fisch — so it never pushes down. - Anything not recognised here simply is not pushed down; `apply_row_scope` still applies the - whole tree, so the wall is unchanged either way. Belt AND braces, deliberately: the pushdown - is what makes the VALUES right, `permits()` is what makes the ROWS right. - - ⭐ THE `bus` FALLBACK, AND WHY IT IS NOT A HOLE IN AMENDMENT 4. On a topic outside - `BU_FILTERABLE_MODULES` there is no column a BU condition could be written against, so - "the filter pins no team" cannot mean "the admin chose consolidated" — it is the only answer - the filter language has. Resolving that silence as None returns the WIDER scope, which makes - the current code fail-OPEN on the values axis for exactly the topic that cannot argue back. - So the record's own `bus` answers instead, and the direction is what makes it safe: this can - only ever REPLACE None (both units) with a pinned single unit. It never widens, it never - touches `may_access`, and a `bus:'all'` account is unaffected because `scope_team_id` returns - None for it — which is every account in tenant #0's registry except the one this shipped for. - """ - if perms.is_admin(user): - return None, None - team_id, agent = _derive_from_filter(user, module) - if team_id is None and module not in BU_FILTERABLE_MODULES: - team_id = perms.scope_team_id(user) - return team_id, agent - - -def _derive_from_filter(user, module): - """`(team_id, agent)` the PERMANENT FILTER pins, before any fallback. Split out so the - fallback has exactly one place to apply — the three exits below all mean "the filter pinned - nothing", and a rule written at each of them is a rule that will one day be written at two.""" - e = entry(user, module) - tree = (e or {}).get('filter') - if not tree: - # Un-migrated records still answer through the legacy derivation, so the old wall keeps - # working until the migration has run. - if not is_migrated(user): - return perms.scope_team_id(user), perms.scope_agent(user) - return None, None - - from harness import filter_eval as fe - nodes, conj = fe.tree_parts(tree) - if conj == 'or': - return None, None - - team_id, agent = None, None - for n in nodes: - if not isinstance(n, dict): - continue - if isinstance(n.get('children'), list): - # A top-level OR GROUP under an AND root IS a guaranteed narrowing — every row must - # satisfy it — so it may push down, unlike a leaf under an OR ROOT (refused above). - # This is the shape a multi-value BU condition actually takes; see `_group_dba_team`. - tid = _group_dba_team(n) - if tid is not None: - team_id = tid if team_id in (None, tid) else None - continue - if n.get('op') not in _PUSHDOWN_OPS: - continue - col = n.get('colId') - raw = n.get('value') - if col == 'dba': - vals = {v.strip().lower() for v in str(raw or '').split(',') if v.strip()} - if not vals: - continue - for allowed, tid in _DBA_TEAM: - if vals <= allowed: - # Both BUs named = no narrowing to push; leave it to the post-filter. - team_id = tid if team_id in (None, tid) else None - break - elif col == 'agent': - v = str(raw or '').strip() - # A SET of agents cannot become the pool's single `agent_name`; the post-filter - # handles it. Only an unambiguous single value pushes down. - if v and ',' not in v: - agent = v - return team_id, agent - - -def _group_dba_team(group): - """The team a top-level `or` group pins, or None. - - Recognises ONLY the exact shape "every child is a `dba eq ` leaf" — the group the - condition builder emits for a multi-value BU condition, and the one `perm_migrate` writes. - Every OTHER group returns None and is left entirely to the post-filter: a group mixing `dba` - with another column, or containing a nested group, does not pin a BU on its own, and - guessing that it does would build the pool from the wrong book. Narrow by construction — - the pushdown may only ever be an OPTIMISATION of a constraint the filter already expresses. - """ - if group.get('conj') != 'or': - return None - children = group.get('children') or [] - if not children: - return None - vals = set() - for c in children: - if (not isinstance(c, dict) or isinstance(c.get('children'), list) - or c.get('colId') != 'dba' or c.get('op') != 'eq'): - return None - v = str(c.get('value') or '').strip().lower() - if not v: - return None - vals.add(v) - for allowed, tid in _DBA_TEAM: - if vals <= allowed: - return tid - return None - - -# ── C1: THE ONE DOOR TO ANY DATABASE'S ROWS (wave 36, W36-T20) ──────────────────────────────── -#: ⭐⭐ OWNER RULING R6, AND IT IS WHY THIS SECTION EXISTS AT ALL: *"EVERY database gets the same -#: permission logic, always"* — per-user field visibility AND row filtration on every database -#: carrying a unique id, whatever created it, with a NEW database inheriting it by construction -#: rather than by a list somebody maintains. -#: -#: ⛔ THE PRODUCT HAD TWO PERMISSION SYSTEMS AND ONLY ONE WAS ARMED. Everything above this line -#: walls the REGISTRY topics (`customer_data`, `product_data`) and is called only from the topic -#: assemblies. Every OTHER database is a `ut_*` table walled by `user_tables.may_open` alone — -#: creator, admin, or a `core.shares` grant — which is a BINARY door: you see all 31,418 rows of -#: `ut_odoo_invoices` or none of them. `perms.tenant_governable_modules`' docstring booked this -#: work in as many words (*"Arming `perm_scope` over `ut_*` … booked, not faked"*), and owner -#: item 11 is that booking coming due. -#: -#: ⚠ AND THE PREMISE THE GRILL GOT WRONG, because the fix depends on it: those databases are NOT -#: user-created. Ten of them (`ut_odoo_invoices`, `…_orders`, `…_agents`, `…_accounts`, `…_bills`, -#: `…_vendors`, `…_order_lines`, `…_gl_lines`, `…_customers`, `…_products`) are generated by the -#: KEYCHAIN connector (`aios-web/api/odoo_relational.py`). **`ut_` is a storage prefix, not a -#: statement about origin**, and a wall keyed off it was reading a naming artefact as a security -#: boundary. -#: -#: ⛔⛔ THE TWO QUESTIONS STAY TWO QUESTIONS. `may_open` answers *"IF you see this database"* and -#: is untouched by this section; C1 answers *"WHICH rows and fields"*. `may_read` below COMPOSES -#: them — it calls `may_open`, it does not reimplement it — because merging them is how this -#: codebase got two ideas of who owns a table once already (`user_tables.may_open`'s own wave-20 -#: note). One resolver per question, asked in order. - - -class UnknownTable(LookupError): - """No database in this tenant answers to that key. - - ⛔ RAISED, NEVER RETURNED AS AN EMPTY LIST (contract C1). An empty list reads as *"this - database is empty"* — indistinguishable from a real empty table, and the caller least able to - notice is the one that wanted rows. This repo has shipped that exact silent-empty answer - before (`user_tables.all_defs`' own correction note; [[empty-answer-vs-unfinished-answer]]). - """ - - -class Denied(PermissionError): - """This principal may not read this database at all. The IF question, answered by `may_read`.""" - - -class Unresolvable(RuntimeError): - """The rows exist and cannot be served under this call's constraints — R6's SECOND SENTENCE. - - ⭐ STANDING RULE 1 IS TWO SENTENCES AND THE SECOND IS THE HALF THAT GETS DROPPED: *"if there - is lag or it can't be done, you need to explicitly tell me why and recommend a fix"*. So a - limit that genuinely cannot be removed is REPORTED with its cause and a recommendation, never - silently enforced as a short answer. Carries the same four keys - `routes_tables._PID_SCOPE_LIMIT` already puts on the wire, so a route can hand this straight - to a client without a second vocabulary ([[one-question-two-normalizers]]). - """ - - def __init__(self, subject, effect, cause, recommendation): - self.subject, self.effect = subject, effect - self.cause, self.recommendation = cause, recommendation - super().__init__(f"{subject}: {effect}. {cause}. {recommendation}") - - def as_limit(self): - """The dict shape `routes_tables` puts in an assembly's `limits` list.""" - return {"subject": self.subject, "effect": self.effect, - "cause": self.cause, "recommendation": self.recommendation} - - -#: Row readers DECLARED by the app layer, keyed by EXACT database key. -#: `reader(table_key, user, st) -> (fields, rows)`. -#: -#: ⛔ WHY A REGISTRY AND NOT AN IMPORT. `core` never imports up (`platform/ARCHITECTURE.md`), and -#: a registry TOPIC's rows are built by `modules/` + `aios_grid` behind an API-layer pool cache -#: (`routes_customers._pool_for`), which is two layers above this file. Same idiom `user_tables` -#: already uses for exactly this reason — `register_connected`, `register_read_through`, -#: `ROW_HOOKS`: *"`core` never imports up, so the app tells this layer rather than being -#: interrogated by it."* -_ROW_SOURCES = {} - -#: THE reader for a read-through `ut_*` grid — one reader, because there is one mirror. -#: `reader(table_key, field_keys, st) -> rows`. -_MIRROR_READER = None - - -def register_rows(reader, *table_keys): - """Declare who reads a NAMED database's rows. Returns the registered key set. - - ⚠ The return value is the registrar's own answer on purpose: a public function whose only - caller is a `verify_*.py` file is a feature no user can reach, and this repo has a gate that - says so ([[reachable-is-not-the-same-as-built]]). Routing the read door through the write - door's return keeps one construction site of the set instead of two. - """ - for key in table_keys: - k = str(key or '').strip() - if k: - _ROW_SOURCES[k] = reader - return frozenset(_ROW_SOURCES) - - -def register_mirror(reader): - """Declare THE reader for read-through `ut_*` grids (`routes_tables._read_through_rows`).""" - global _MIRROR_READER - _MIRROR_READER = reader - return _MIRROR_READER is not None - - -#: ⛔ `row_sources()` IS DELETED (W36-T24 / owner item 13), AND THE REASON IS THE ONE THIS WAVE -#: KEEPS FINDING. It returned `frozenset(_ROW_SOURCES)` under a docstring calling itself *"The ONE -#: list to read"* — and `register_rows` ALREADY returns exactly that, which is the same idiom -#: `user_tables.register_connected_prefix` uses and the same reason: routing the read door through -#: the write door's return keeps ONE construction site of the set. A second accessor beside it is a -#: parallel path with nothing of its own to say, and it shipped with no caller outside `verify_*.py` -#: — the shape that is whole, correct and unreachable ([[artifact-with-no-importer]]; reported by -#: the integrator's `web_reachability` pass, `mailbox/A.md` A-43). The registrar's return is the -#: read: `routes_grid._C1_ROW_SOURCES` is that value, held where it is registered. - - -def _ut(): - import core.user_tables as user_tables - return user_tables - - -def may_read(user, table_key, st=None): - """May this principal read `table_key` AT ALL — the IF question, on EVERY database. - - ⛔ COMPOSED, NOT RE-DERIVED, and the order is the whole rule: - - 1. an admin reads everything (break-glass — `deps._user_for` hands back a hardcoded master - dict on a store outage and it will never carry a `perms` block); - 2. an EXPLICIT stored `access: false` DENIES, on any database. This is the toggle owner - item 11 asks for, and it is a **deny-only overlay**: it may revoke a database the wall - below would admit, and it may never grant one that wall refuses; - 3. a `ut_*` database defers to `user_tables.may_open` — creator, admin, or a `core.shares` - grant — UNMODIFIED. W36-T21: *"`may_open` still decides IF the database is visible."* - 4. anything else is a registry topic and defers to `may_access` above. - - ⛔⛔ WHY ABSENCE MUST NOT DENY ON A `ut_*` KEY, which is the opposite of what leg 4 does. - `may_access` reads migrated-and-undeclared as DENY — correct for a topic, because - `routes_admin` writes an entry for every governable topic on every save. ⚠ NO `ut_*` ENTRY WAS - STORABLE AT ALL UNTIL W36-T22 — `_clean_perms` refused the key with `unenforced_module` — so - every record migrated before this wave carries no entry for any of them, and reading that - absence as a decision would revoke all ten keychain databases from every migrated account the - moment this arms. That is not R6, it is an outage. Leg 3 therefore asks the wall that HAS been - answering rather than the marker that has not, and it keeps being right AFTER the flag's - deletion: an admin who has never opened the editor for a database has still not decided - anything about it. - - ⛔⛔ PASS THE **PUBLIC** RECORD, NOT THE ONE OUT OF `users.json`. Leg 3 needs a username, and a - stored record is keyed BY username in that bucket and does not carry one INSIDE it — only - `core.users._public(uname, rec)` puts it there, which is what `deps.Session.user` holds. Hand - this the raw record and `may_open` gets a `None` viewer and fail-closes, so every `ut_*` - database reads as DENIED for an account that can open all of them. It fails in the SAFE - direction and is silently wrong, which is the worst pair to debug — it cost two call sites in - one afternoon: a gate double, and `routes_admin.get_perms`' own fix for this very outage. - """ - if perms.is_admin(user): - return True - e = entry(user, table_key) - if e is not None and not bool(e.get('access', True)): - return False - key = str(table_key or '') - if key.startswith(_ut().KEY_PREFIX): - return bool(_ut().may_open(key, (user or {}).get('username'), False, st=st)) - return may_access(user, table_key) - - -def wall_declared(user, table_key): - """Is a ROW or FIELD narrowing declared for this principal on this database? - - ⛔ THE QUESTION A DOOR ASKS BEFORE SERVING ROWS IT CANNOT SCOPE. `perms.py` warned that a - stored `ut_*` wall would be INERT — *"the editor would say DENY, the table routes would keep - serving, and nothing anywhere would say so"*. A route that cannot apply C1 must therefore - REFUSE for a principal this returns True for, rather than serve the whole database. False for - an admin (they bypass the wall entirely) and for any record with no entry, so a door asking - this pays nothing and changes nothing for everybody who has no wall. - """ - if perms.is_admin(user): - return False - e = entry(user, table_key) - if not e: - return False - return bool(e.get('filter')) or bool(e.get('hiddenFields')) - - -def row_scope_applies(user, table_key): - """Does a permanent ROW filter narrow this principal on this database? - - ⚠ `wall_declared`'s narrower half, and it exists so a rows-free caller can SKIP building rows - it would only need in order to filter them. `routes_tables.scoped_pids` is that caller: its - whole point is that the pid set costs no row pass, and paying for one on every database - switch — for every account, walled or not — would undo W30-T30 to enforce a rule that applies - to almost nobody. Asked here rather than spelled out at the call site, so there is ONE - statement of when the row wall bites ([[one-question-two-normalizers]]). - """ - if perms.is_admin(user): - return False - return bool((entry(user, table_key) or {}).get('filter')) - - -def scoped_table(user, table_key, st=None, ctx=None): - """⭐⭐ CONTRACT C1 — the rows of ANY database, already field-stripped and row-filtered for - `user`. Registry topic or `ut_*`; there is no third kind and no per-database branch. - - rows = scoped_table(user, 'ut_odoo_invoices') # a keychain database - rows = scoped_table(user, 'customer_data') # a registry topic - - `user` is a user RECORD (the dict `deps.Session.user` carries), not a username — the whole - wall is a pure function of that record. BOTH arguments are positional and REQUIRED: a caller - that forgets the principal must not run, because the only thing a defaulted one could mean is - "unscoped", which is the widening direction. - - ⛔ FAIL-CLOSED, THREE WAYS, AND EACH IS A DIFFERENT EXCEPTION so a caller can answer with the - right status instead of guessing: `UnknownTable` (no such database — never an empty list), - `Denied` (the IF question said no), `Unresolvable` (the rows cannot be served and here is - why — standing rule 1's second sentence). - - ⚠ NO CAP. A connected source is read THROUGH the mirror in full (standing rule 1); the only - thing that stops it is a population that exceeds one materialisation window, and that arrives - as `Unresolvable` carrying its cause and a recommendation rather than as a short answer. - - ⭐ E's SCRIPT SANDBOX HOLDS NO SECOND PATH TO THE STORE (wiring W1), which is why this is - THE door rather than A door: everything a sandboxed script may read, it reads here, under the - CALLING user's scope (R5). - """ - _fields, rows = _scoped(user, table_key, st=st, ctx=ctx) - return rows - - -def scoped_fields(user, table_key, st=None): - """The COLUMNS of any database this principal may see — C1's other half. - - ⛔ IT IS NOT A CONVENIENCE, IT IS THE SECOND WIRE. `strip_row`'s own note above says it: the - field list and the row payload are two different wires, and narrowing one without the other - leaves the value sitting where anything can read it. A caller that must render a scoped table - needs both, and E cannot read a `ut_*` definition to learn its columns — the sandbox has no - second path to the store (W1). So both come from here, off one wall. - - ⚠ On a `ut_*` database this reads the DEFINITION only — the projection, no rows (D-213). On a - registry topic it goes through the registered reader, which builds that topic's pool; the - pool is cached per scope on the tenant runtime, so it is a cache hit next to `scoped_table`. - """ - fields, _rows = _scoped(user, table_key, st=st, ctx=None, want_rows=False) - return fields - - -def _scoped(user, table_key, st=None, ctx=None, want_rows=True): - """`(fields, rows)` — ONE evaluator behind both public doors, so they cannot disagree.""" - key = str(table_key or '').strip() - if not key: - raise UnknownTable('a database key is required. This door will not guess which database ' - 'was meant') - if not may_read(user, key, st=st): - raise Denied(f"this account may not read '{key}'") - fields, rows = _read(key, user, st, want_rows) - # THE FIELD WALL — a TRANSITIVE closure, so hiding a column also hides every formula computed - # FROM it. Resolved ONCE and used for both wires; see `hidden_keys` for why a set difference - # is the wrong shape here. - # ⭐ W38-T16 — `st` RIDES INTO THE WALL, not just into the read. The field-grant leg resolves - # against `object_shares` in THIS tenant's store; without the handle it would answer from the - # module default (tenant #0) and hide a grantee's own column on every other tenant. - hide = hidden_keys(user, key, fields, st=st) - if not want_rows: - return (visible_fields(fields, user, key, st=st) if hide else fields), [] - # THE ROW WALL — `permits()`, so a permanent filter this evaluator cannot answer DENIES - # rather than being ignored. Evaluated against the UNSTRIPPED contract on purpose: a - # permanent filter may name a column the reader is not allowed to SEE, and dropping the - # predicate would widen the read rather than narrow it. - rows = apply_row_scope(rows, user, key, fields, ctx) - if hide: - fields = visible_fields(fields, user, key, st=st) - rows = [strip_row(r, hide) for r in rows] - return fields, rows - - -def _read(table_key, user, st, want_rows=True): - """`(fields, rows)` BEFORE the wall — the app layer's reader, or core's own for a `ut_*`.""" - reader = _ROW_SOURCES.get(table_key) - if reader is not None: - fields, rows = reader(table_key, user, st) - return list(fields or ()), list(rows or ()) - ut = _ut() - if not table_key.startswith(ut.KEY_PREFIX): - # ⛔ A TOPIC WITH NO REGISTERED READER IS UNKNOWN, NOT EMPTY. In a process that never - # imported the API layer this is the honest answer: nothing here can build that pool. - raise UnknownTable(f"no database named '{table_key}' in this workspace, and no reader " - f"is registered for it") - return _read_user_table(table_key, st, want_rows) - - -def _read_user_table(table_key, st, want_rows=True): - """core's OWN reader for a `ut_*` database. Answers with NO registrar, deliberately. - - ⭐ WHY IT LIVES IN `core` RATHER THAN BEING REGISTERED LIKE THE TOPICS, and it is the same - argument that seeds `user_tables._CONNECTED_PREFIXES` rather than registering it: a cold - process — E's sandbox subprocess, a worker, a gate — that never imported an API route still - owes the right answer for `ut_odoo_invoices`. A registrar-only design would raise there, and - the sandbox is exactly such a process. - - ⚠ THE WALL IS ANSWERED ON A PROJECTION AND THE ROWS ARE NOT. `lend_defs` serves definitions - without the 28.6 MB of rows (D-213), which is every read this function makes when - `want_rows` is false; a projected document RAISES on `rows` rather than answering empty, so - the materialised arm below takes the whole read explicitly. - """ - ut = _ut() - lent = ut.lend_defs(st) - defn = ut.get(table_key, st=lent) - if not defn: - raise UnknownTable(f"no database named '{table_key}'") - fields = [dict(f) for f in (defn.get('fields') or [])] - if not want_rows: - return fields, [] - if not ut.materialises(table_key, st=st, defn=defn): - # A read-through grid stores no rows here — they live in the mirror, and reading - # `defn['rows']` would find an empty dict and serve an EMPTY GRID: correct-looking, - # wrong, and silent. - if _MIRROR_READER is None: - raise Unresolvable( - subject='rows', effect='unreadable', - cause=(f"'{table_key}' is served read-through from the connector mirror and no " - f'mirror reader is registered in this process'), - recommendation=('call `perm_scope.register_mirror(...)` from the app layer before ' - 'reading a read-through database, or read it through the API')) - keys = {f['key'] for f in fields if f.get('key')} - return fields, list(_MIRROR_READER(table_key, keys, st) or ()) - whole = ut.get(table_key, st=st) - if whole is None: - # Deleted between the wall and here. The same refusal, not an empty table. - raise UnknownTable(f"no database named '{table_key}'") - field_keys = {f['key'] for f in fields if f.get('key')} - rows = [] - for rid, row in (whole.get('rows') or {}).items(): - if not str(rid).isdigit(): - continue - r = {k: v for k, v in (row or {}).items() if k in field_keys} - r['pid'] = int(rid) - rows.append(r) - rows.sort(key=lambda r: r['pid']) - return fields, rows +"""core/perm_scope.py — the permission WALL for table modules (wave 15, C-PERM). + +ONE place answers the three questions a restricted account raises on a grid surface: + + may_access(user, module) may they open it at all? + visible_fields(fields, u, mod) which COLUMNS may they receive? + apply_row_scope(rows, u, mod, …) which ROWS may they receive? + +plus one that exists only because of how the pool is built: + + derive_pool_scope(user, module) which (team_id, agent) must the pool be BUILT with? + +Both hosts call these — `aios-web/api` (`grid_assembly`) and `app.py` (`_table_grid`) — because +a wall that exists on one runtime and not the other is not a wall. `core/perms.py` stays what it +is (module GRANTS + the legacy BU derivation); this module is the row/field/pushdown layer that +sits on top, and it is deliberately a separate file so the legacy readers can keep their +semantics untouched while this one fails closed. + +──────────────────────────────────────────────────────────────────────────────────────────── +THE RECORD + + user['perms'] = {'': {'access': bool, + 'filter': {'conj'?: 'and'|'or', 'nodes': [...]} | None, + 'hiddenFields': ['', ...]}} + user['perms_v'] = 1 stamped by the migration and by every write + +`perms_v` is the EXPLICIT-RESOLUTION marker, and it is here because of `permissioning.md` +Part II gap #5: "`allowed_modules() is None` is fail-open by default … make 'resolved: +unrestricted' an explicit value so absence/uncertainty DENIES." The same class already shipped +twice in this codebase (`modules: []` and `bus: []` both read as UNRESTRICTED — +`routes_admin.py`'s own docstring documents both). So: + + * `perms_v` ABSENT → the record is UN-MIGRATED, and the LEGACY wall applies unchanged + (`core.perms` module grants + the `bus`/`agent` query scope). That is not fail-open: it is + today's real wall, and it bounds the rollout window to "until the migration runs". + * `perms_v == 1` and the module has NO entry → **DENY**. Absence now means what it says. + * `role == 'admin'` bypasses all of it — which is also what keeps BREAK-GLASS alive. + `deps._user_for` hands back a hardcoded master dict on a store outage + (`{'username':'admin','role':'admin','bus':'all','modules':'all'}`) that will never carry a + perms block; without this clause an explicit-marker scheme locks the owner out of their own + product at exactly the moment the store is broken. +""" +import re + +import core.perms as perms + +#: `aios_grid._FORMULA_REF`'s pattern, restated rather than imported: this module is imported by +#: the API's request path and `aios_grid` pulls in the whole grid stack. Same regex, one line, +#: and `verify_api` asserts the two agree so it cannot drift into a different grammar. +_FORMULA_REF = re.compile(r"\{([^{}]*)\}") + +PERMS_VERSION = 1 + + +def _rec(user): + return user if isinstance(user, dict) else {} + + +def is_migrated(user): + """True once this record carries an explicit resolution. See the module docstring.""" + return int(_rec(user).get('perms_v') or 0) >= PERMS_VERSION + + +def entry(user, module): + """This user's declared permissions for `module`, or None if nothing is declared. + + None is AMBIGUOUS on purpose and every caller must resolve it against `is_migrated`: + on a migrated record it means DENY, on a legacy one it means "ask the old wall". + """ + p = _rec(user).get('perms') + if not isinstance(p, dict): + return None + e = p.get(module) + return e if isinstance(e, dict) else None + + +def may_access(user, module): + """May this account open `module` at all? Fail-closed on a migrated record.""" + if perms.is_admin(user): + return True + e = entry(user, module) + if e is not None: + return bool(e.get('access', True)) + if is_migrated(user): + # Migrated and undeclared = denied. This is the whole point of the marker. + return False + return perms.may_open(user, module) # legacy record: the old grant wall + + +def may_metrics(user, module): + """May this account build and receive METRIC columns — lookback measures — on `module`? + + ⭐⭐ W38-T19 — A CAPABILITY, NOT A SECOND SPELLING OF ACCESS, and the distinction is the + ticket. A rollup aggregates the CHILDREN a row is linked to; a Metric answers *"this number, + over this window"* against a governed topic with no relation at all (CLAUDE.md standing rule + 9). So it reads the book behind the rows rather than the rows: an account can be exactly the + right person to see a customer list and the wrong one to mint 12-month revenue over it. Two + decisions, two controls. + + ⛔ ABSENCE GRANTS, AND THE ASYMMETRY WITH `may_access` IS DELIBERATE RATHER THAN AN + OVERSIGHT TO TIDY. `may_access` reads migrated-and-undeclared as DENY, which is right there + because every save writes an entry for every governed database — absence means an + administrator decided. No `metrics` KEY was STORABLE before this ticket, so every migrated + record in every tenant carries none, and reading that absence as DENY would revoke Metrics + for everybody on the day this shipped with nobody having decided anything. That is the exact + failure `routes_admin.get_perms` records twice already (the `ut_*` default and the surface + default), one field further down the same entry. **Only an explicit `metrics: false` + refuses.** + + ⚠ IT DOES NOT RE-ASK ACCESS. Every caller is already behind the access wall (`module_gate`, + `may_read`, `session.require`), and folding admission in here would give a denial two + possible causes with one answer — the shape that makes a permission bug take an afternoon. + """ + if perms.is_admin(user): + return True + e = entry(user, module) + return not (isinstance(e, dict) and e.get('metrics') is False) + + +def nav_may_open(user, key, st=None): + """May this account SEE `key` — in the nav, and at the route gate? The ADMISSION question. + + ⛔⛔ WHY THIS EXISTS: THE EDITOR'S DECISION REACHED THE READ DOOR AND NOTHING ELSE. Measured + on live 2026-08-18, on a real account. An administrator ticked *Odoo products* for Naomi in + Manage user and saved; the record stored `perms.product_data.access = True` and + `may_access()` agreed. **The database never appeared.** `perms.nav_pages` and + `deps.Session.require` both ask `perms.may_open` — the LEGACY `user['modules']` array — which + still read `['sales', 'customers', 'products']`, and `products` is the key of the ARCHIVED + *SKU* module, not of `product_data`. So `nav_pages` answered `['customer_data']` and the + route would have 403'd her even by URL. Two permission systems, the editor writing one and + every DOOR reading the other ([[two-permission-systems-one-armed]]). + + ⭐ THE ASYMMETRY IS THE WHOLE DESIGN, and it is not the same rule twice: + + 1. an admin sees everything (break-glass, as everywhere else); + 2. an EXPLICIT `access: false` DENIES, on any key — an administrator unticking a box must + take the row off the nav, and before this it did not; + 3. a `ut_*` key with no explicit deny defers to `user_tables.may_open` — creator, admin or + a `core.shares` grant — and **an `access: true` entry may NEVER widen past it**. The + editor must not become a way to hand somebody another user's private table; + 4. a REGISTRY TOPIC with an explicit entry takes that entry, grant included. Here the + editor IS the authority: `_clean_perms` writes an entry for every governed topic on + every save, so an entry means an administrator decided; + 5. anything else — a module this editor does not govern — falls through to + `perms.may_open`, UNCHANGED. + + ⛔ LEG 5 IS LOAD-BEARING AND IT IS WHY `may_access` COULD NOT SIMPLY BE CALLED HERE. + `may_access` reads migrated-and-undeclared as DENY, which is correct for a governed topic and + catastrophic for the nav: Naomi's block declares the ten governed keys and nothing else, so + `sales` would have gone from visible to denied — an outage dressed as a permission fix. + """ + if perms.is_admin(user): + return True + e = entry(user, key) + if e is not None and not bool(e.get('access', True)): + return False + if str(key or '').startswith(_ut().KEY_PREFIX): + return bool(_ut().may_open(key, (user or {}).get('username'), False, st=st)) + if e is not None: + return bool(e.get('access', True)) + return perms.may_open(user, key) + + +def assistant_entry(user, module): + """The explicit database grant an Assistant snapshot may rely on, else ``None``. + + The interactive application retains an admin break-glass path and a temporary legacy-grant + compatibility path. Neither is an answer at the Assistant's app-stored data boundary: + that reader must be able to name the migrated grant that admitted a database. In particular, + a store-outage admin identity with no ``perms`` document is not an unresolved permission that + may be widened into data access. + """ + e = entry(user, module) + if (not is_migrated(user) or not isinstance(e, dict) + or not bool(e.get('access', True)) or not may_access(user, module)): + return None + return e + + +# ── FIELDS ─────────────────────────────────────────────────────────────────────────────────── +#: ⭐⭐ W38-T16 — THE MARKER THAT SAYS "THIS COLUMN IS GOVERNED BY A GRANT", stamped once by the +#: door that creates a shared column (`routes_tables.patch_shared_cell`) and never flipped by any +#: door afterwards. It is the `perms_v` idiom one object down, and it is here for the reason that +#: marker exists: **absence must not be read as a decision.** +#: +#: ⛔⛔ WHY A MARKER AND NOT "DOES A GRANT RECORD EXIST". Every tenant-wide column shipped before +#: this ticket carries no `field` grant record, because none was STORABLE — `shares.KINDS` had +#: three members. Reading that absence as "granted to nobody" would blank every existing shared +#: column for every account the moment this arms, which is not a permission fix, it is an outage +#: (`may_read` leg 3 carries the same argument for `ut_*` keys, in the same words). +#: +#: ⛔⛔ AND IT IS WHAT MAKES THE WALL FAIL **CLOSED**. The polarity here is the opposite of every +#: other grant check in this repo: a field share is a GRANT, so "no grant" has to mean HIDDEN or +#: the wall does nothing — but "no grant record" also describes a legacy column and a store that +#: could not be read. The marker separates the three: `granted` on the column means an explicit +#: decision was taken, so an unreadable registry hides it; no marker means legacy, so nothing +#: changes. Without it, one unreadable read of `object_shares` would publish every governed +#: column to the whole tenant, silently ([[a-guard-for-the-dangerous-case]]). +#: +#: ⚠ IT IS NOT `shared_overlay`'s `"shared": True` AND MUST NOT BE CONFUSED WITH IT. That flag is +#: written and never read (D-414) — `is_shared` is a dict-membership test — so it is not evidence +#: of anything. This one is read HERE, on every assembly, and the only writer is the create door. +FIELD_GRANT_MARK = 'granted' + + +def granted_field_keys(fields): + """Every column in `fields` that declares itself GOVERNED by a `shares` grant. + + ⭐ THE CHEAP HALF OF THE WALL, AND IT IS WHY THE WALL COSTS NOTHING FOR ALMOST EVERYBODY. It + is a scan of dicts already in memory, so a database with no governed column reaches no store + at all and `hidden_keys` behaves exactly as it did before this ticket. The registry is only + opened once this answers non-empty. + """ + return {str(f['key']) for f in (fields or ()) + if isinstance(f, dict) and f.get('key') + and f.get(FIELD_GRANT_MARK) is True} + + +def field_grant_hidden(user, table_key, fields, st=None): + """The governed columns of `table_key` this principal holds NO grant on — C1's per-FIELD wall. + + ⭐⭐ W38-T16 / R7 / R8 — THE THIRD WALL, AND IT COMPOSES ALONGSIDE THE OTHER TWO RATHER THAN + INSIDE THEM. `may_open` answers *IF* you reach a database; `may_read`'s stored `access: false` + overlay may REVOKE one; this answers *WHICH COLUMNS* of it you receive. It is deliberately not + threaded through that overlay: the overlay is **deny-only** by its own docstring (*"it may + revoke a database the wall below would admit, and it may never grant one that wall refuses"*) + and a field share is a GRANT — the widening direction. Merging them would give the codebase a + second idea of who grants what, which is the failure `may_open`'s own note is the record of. + It composes the way `may_open`'s grant leg does: additively, last, fail-closed. + + ⛔ AN ADMIN IS NOT WALLED (break-glass, as everywhere else) and neither is the column's OWNER — + `shares.role_for` answers `'owner'` for the creator, so a person cannot lose their own column + by forgetting to share it with themselves. + + ⚠ `st` IS THE TENANT HANDLE AND ITS ABSENCE IS SAFE HERE, unlike everywhere else. A caller + that cannot lend one reads the module-default bucket; on any tenant but #0 that finds no + grant, and no grant on a MARKED column means HIDDEN. So a door that has not learned to thread + `st` under-shares rather than over-shares, and the symptom is a grantee who cannot see their + column — visible, reportable, and the opposite of a leak. + """ + if perms.is_admin(user): + return set() + marked = granted_field_keys(fields) + if not marked: + return set() + uname = str((user or {}).get('username') or '').strip().lower() + try: + import core.shares as shares + except Exception: # noqa: BLE001 + return set(marked) + if not uname: + # No principal, and a marked column is an explicit decision: nobody is not somebody. + return set(marked) + # ⭐ THE CREATOR IS READ OFF THE COLUMN, NOT OUT OF THE REGISTRY, AND THAT IS NOT A SECOND + # AUTHORITY. `createdBy` is ALREADY what decides who may DELETE a shared column (R8 / D-172, + # `routes_tables.delete_shared_field`) and who may CLAIM it (`routes_shares._owns_object`); + # asking the same field here keeps one answer to "whose column is this" across all three. + # ⛔ IT IS ALSO THE ONLY THING THAT SURVIVES A CLAIM THAT NEVER LANDED. The create door writes + # the definition first and the grant record second, on purpose — so the window where a column + # is marked and unclaimed exists, and without this line its own author would be walled out of + # the column they just made, permanently and with no way to fix it but an admin. + mine = {str(f['key']) for f in (fields or ()) + if isinstance(f, dict) and f.get('key') and str(f['key']) in marked + and str(f.get('createdBy') or '').strip().lower() == uname} + hide = set() + for key in marked - mine: + try: + oid = shares.field_oid(table_key, key) + role = shares.role_for('field', oid, uname, is_admin=False, st=st) + except Exception: # noqa: BLE001 + role = None + if role is None: + hide.add(key) + return hide + + +def hidden_keys(user, module, fields, st=None): + """The TRANSITIVE closure of hidden field keys (C-PERM amendment 5). + + ⛔ WHY A CLOSURE AND NOT A SET DIFFERENCE. A formula field is computed in the BROWSER + (`formulaEngine.ts`, injected by `computedRows`) from `{ref}`s into other columns, and a + measure column's value arrives precomputed in `derived`. So hiding field X has exactly three + possible outcomes and only one of them is coherent: + + strip X, keep formulas → every formula over X computes blank or wrong, silently + keep X's value for them → X has leaked, wearing a formula's name + strip X AND its dependents→ the only honest answer + + So a hidden field drags every formula that references it — and every formula that references + THAT formula, hence the fixpoint loop — out of the payload with it. + + ⚠ This runs on every assembly, so it is a fixpoint over a handful of custom fields, not a + graph library. `MAX_PASSES` bounds a reference cycle the client would refuse to evaluate + anyway; without it a self-referential pair would spin here. + + ⭐⭐ W36-T21 — AND A ROLLUP IS THE SAME LEAK ONE MECHANISM OVER, which matters now that this + closure runs on the `ut_*` databases rather than only on the two registry topics. A rollup + names a LINK COLUMN OF THIS TABLE (`rollup.link`) and aggregates a field on the table that + link points at — so `ut_odoo_customers.ar_outstanding` is *"sum `residual` over the invoices + this row links to"*. Hide `invoices` and keep `ar_outstanding` and the reader still learns + what the hidden link contains, in aggregate; the three outcomes are exactly the three the + formula argument above enumerates, and only "strip both" is coherent. Verified against the + real declarations (`odoo_relational.customer_fields`) rather than assumed: every rollup there + is either `{'link': , 'field': }` + or a `source` topic aggregate, so `rollup.link` is the ONE same-table reference a rollup makes + and `rollup.field` is deliberately not treated as one — it names another database's column, + which has its own wall. + """ + if perms.is_admin(user): + return frozenset() + # ⭐⭐ W38-T16 — TWO SOURCES OF HIDING, ONE CLOSURE, AND THAT UNION IS THE WHOLE INTEGRATION. + # + # The administrator's `hiddenFields` and a field's own grant answer different questions and + # both end in the same place: a key this reader may not receive. Seeded together HERE, before + # the fixpoint, so the transitive argument above covers the new source unchanged — a formula + # (or a rollup) over a column this reader was not granted comes out with it, or the value + # leaks wearing the derived column's name. + # + # ⛔ AND THIS FUNCTION IS THE INSERTION POINT RATHER THAN `_scoped`, WHICH IS WHAT THE TICKET + # ASSUMED. `_scoped` is C1's evaluator and reaches C1's two doors; **the product reads through + # neither of them.** Every grid door calls THIS: `routes_customers:196`, `routes_products:345`, + # `routes_odoo_tables:920/1119`, `routes_tables._ut_hidden/_ut_field_wall`, `routes_slack:190`, + # `routes_grid:57/69/787` — and `grid_events` walls WRITES off the same set through + # `EventCtx.hidden_keys`. One evaluator, every door, both wires, read and write. + hidden = set(field_grant_hidden(user, module, fields, st=st)) + e = entry(user, module) + if e: + hidden |= {str(k) for k in (e.get('hiddenFields') or ()) if k} + if not hidden: + return frozenset() + + refs = {} + for f in fields or (): + if not isinstance(f, dict) or not f.get('key'): + continue + expr = f.get('formula') + if isinstance(expr, str) and expr: + refs[f['key']] = {m.strip() for m in _FORMULA_REF.findall(expr) if m.strip()} + link = (f.get('rollup') or {}).get('link') if isinstance(f.get('rollup'), dict) else None + if isinstance(link, str) and link.strip(): + refs.setdefault(f['key'], set()).add(link.strip()) + + MAX_PASSES = 12 + for _ in range(MAX_PASSES): + grew = False + for key, deps in refs.items(): + if key not in hidden and deps & hidden: + hidden.add(key) + grew = True + if not grew: + break + return frozenset(hidden) + + +def visible_fields(fields, user, module, st=None): + """`fields` minus the hidden closure. Order preserved — the column order is the user's. + + ⚠ W38-T16 — `st` MUST MATCH WHAT ITS CALLER PASSED TO `hidden_keys`, and that is not a style + note. This recomputes the closure, and since the field-grant leg under-hides without a tenant + handle, a caller that lends one to `hidden_keys` and not to this would narrow the FIELD LIST + by MORE than it stripped from the ROWS — the value left sitting in the payload under a column + nobody can see, which is exactly the half-wall `strip_row`'s note exists to forbid. + """ + hide = hidden_keys(user, module, fields, st=st) + if not hide: + return list(fields or ()) + return [f for f in (fields or ()) + if not (isinstance(f, dict) and f.get('key') in hide)] + + +def assistant_visible_fields(fields, user, module): + """Visible closure under one explicit Assistant grant, including for an admin caller.""" + e = assistant_entry(user, module) + if e is None: + return [] + # `hidden_keys` deliberately preserves the interactive admin break-glass behaviour. The + # Assistant uses its explicit grant instead, but shares the same transitive formula closure. + hidden = {str(k) for k in (e.get('hiddenFields') or ()) if k} + if not hidden: + return list(fields or ()) + + refs = {} + for field in fields or (): + if not isinstance(field, dict) or not field.get('key'): + continue + expr = field.get('formula') + if isinstance(expr, str) and expr: + refs[field['key']] = {match.strip() for match in _FORMULA_REF.findall(expr) + if match.strip()} + for _ in range(12): + grew = False + for key, deps in refs.items(): + if key not in hidden and deps & hidden: + hidden.add(key) + grew = True + if not grew: + break + return [field for field in (fields or ()) + if not (isinstance(field, dict) and field.get('key') in hidden)] + + +def strip_row(row, hide): + """Drop hidden keys from ONE assembled row. Cheap enough to run per row, and it must run + per row: the field list and the row payload are two different wires, and stripping only the + first would leave the value sitting in the second where anyone can read it.""" + if not hide or not isinstance(row, dict): + return row + return {k: v for k, v in row.items() if k not in hide} + + +# ── ROWS ───────────────────────────────────────────────────────────────────────────────────── +def apply_row_scope(rows, user, module, fields, ctx=None): + """The rows this account may receive: `filter_eval.permits` over the permanent filter. + + `permits`, never `matches` — an unanswerable permanent filter DENIES rather than being + ignored. See `harness/filter_eval`'s docstring for the field-rename walkthrough that makes + the difference a leak rather than a preference. + """ + if perms.is_admin(user): + return list(rows or ()) + e = entry(user, module) + tree = (e or {}).get('filter') + if not tree: + return list(rows or ()) + from harness import filter_eval as fe + return [r for r in (rows or ()) if fe.permits(tree, r, fields, ctx)] + + +def assistant_apply_row_scope(rows, user, module, fields, ctx=None): + """Apply an explicit Assistant grant's permanent filter without an admin bypass.""" + e = assistant_entry(user, module) + if e is None: + return [] + tree = e.get('filter') + if not tree: + return list(rows or ()) + from harness import filter_eval as fe + return [row for row in (rows or ()) if fe.permits(tree, row, fields, ctx)] + + +def validate_assistant_filter(tree, fields): + """Return a strict, detached Assistant filter tree or raise ``ValueError``. + + The display filter cleaner is intentionally permissive: it drops a stale column or leaves an + inactive condition alone so an old saved view can still open. An Assistant data reader cannot + inherit that behaviour — dropping a predicate turns a request for a subset into a wider read. + This validator therefore admits only visible field operands that the existing evaluator can + answer from one stored row. Cohort, measure and rank conditions need separate materialised + set resolvers and are refused here rather than guessed. + """ + if tree is None: + return None + from harness import filter_eval as fe + + by_key = {str(field.get('key')): field for field in (fields or ()) + if isinstance(field, dict) and field.get('key')} + if not by_key: + raise ValueError('assistant filter has no visible field contract') + + def _leaf(raw): + if not isinstance(raw, dict): + raise ValueError('assistant filter leaf must be an object') + allowed = {'id', 'colId', 'op', 'value', 'value2', 'rhs'} + if set(raw) - allowed: + raise ValueError('assistant filter carries an unsupported operand') + col = raw.get('colId') + op = raw.get('op') + if not isinstance(col, str) or col not in by_key: + raise ValueError('assistant filter names an unknown or hidden field') + if (not isinstance(op, str) or op not in fe.FILTER_OPS or op in fe.RANK_OPS + or op in {'between', 'within'} or col == fe.COHORT_FIELD): + raise ValueError('assistant filter uses an unsupported operator') + rhs = raw.get('rhs') + if rhs is not None: + if (not isinstance(rhs, dict) or rhs.get('kind') != 'field' + or set(rhs) - {'kind', 'colId'} + or not isinstance(rhs.get('colId'), str) + or rhs['colId'] not in by_key): + raise ValueError('assistant filter names an unknown or hidden right-hand field') + out = {name: raw[name] for name in ('id', 'colId', 'op', 'value', 'value2') + if name in raw} + if rhs is not None: + out['rhs'] = {'kind': rhs.get('kind'), 'colId': rhs['colId']} + if not fe.is_rule_active(out, by_key): + raise ValueError('assistant filter is inactive or unanswerable') + return out + + def _node(raw): + if not isinstance(raw, dict): + raise ValueError('assistant filter node must be an object') + if 'children' not in raw: + return _leaf(raw) + if set(raw) - {'conj', 'children'}: + raise ValueError('assistant filter group carries an unsupported operand') + children = raw.get('children') + if raw.get('conj') not in ('and', 'or') or not isinstance(children, list) or not children: + raise ValueError('assistant filter group must be a non-empty and/or group') + return {'conj': raw['conj'], 'children': [_node(child) for child in children]} + + if isinstance(tree, list): + if not tree: + raise ValueError('assistant filter list must not be empty') + return {'conj': 'and', 'nodes': [_node(node) for node in tree]} + if not isinstance(tree, dict) or set(tree) - {'conj', 'nodes'}: + raise ValueError('assistant filters must be a tree with nodes') + nodes = tree.get('nodes') + if tree.get('conj') not in ('and', 'or') or not isinstance(nodes, list) or not nodes: + raise ValueError('assistant filter tree must be a non-empty and/or tree') + return {'conj': tree['conj'], 'nodes': [_node(node) for node in nodes]} + + +# ── THE PUSHDOWN ───────────────────────────────────────────────────────────────────────────── +#: `dba` value sets that pin a BU, and the Odoo team they pin. `_dba_attrs` emits exactly +#: {'Fisch','Royal','Both'} (blank for "no brand attributable in 24 months"), so a permission +#: filter admitting Fisch admits {Fisch, Both} — a Both customer IS a Fisch customer. +_DBA_TEAM = ((frozenset({'fisch', 'both'}), 5), (frozenset({'royal', 'both'}), 6)) + +#: Only equality pushes down. `FILTER_OPS` is single-valued (there is no column-level "is any +#: of" — that vocabulary belongs to cohort leaves and is disjoint), so a multi-value BU +#: condition arrives as an OR GROUP of `eq` leaves, handled by `_group_dba_team` below. +_PUSHDOWN_OPS = frozenset({'eq'}) + +#: ⭐ THE MODULES WHOSE FIELD VOCABULARY CAN EXPRESS A BUSINESS UNIT — i.e. whose canonical +#: contract carries a `dba` column. R1's "BU access is just a permanent filter" holds only on +#: these; on every other topic there is no column to write the condition against, so a filter +#: literally cannot say it and the record's `bus` remains the only place the fact lives. +#: +#: `product_data` is the counter-example that made this constant necessary: 19 fields, no brand +#: column, and a product's BU is a property of WHOSE ORDERS built its revenue rather than of the +#: SKU. Without the fallback below, a Fisch-only account read the product catalogue with +#: Fisch+Royal money on every row — the amendment-3 defect, arriving through the door amendment 3 +#: was written to close. +#: +#: ⚠ A LIST THAT MIRRORS A JSON CONTRACT DRIFTS UNLESS SOMETHING CHECKS IT. `verify_api` asserts +#: membership here matches "this topic's contract has a `dba` field" for every governed module, so +#: a topic that grows or loses a brand column cannot silently keep the wrong rule. Named in this +#: module rather than derived from `aios_grid` on purpose: `perm_scope` is on the API's request +#: path and importing the grid stack to answer a two-element question is a cost per request. +BU_FILTERABLE_MODULES = frozenset({'customer_data'}) + + +def derive_pool_scope(user, module): + """`(team_id, agent)` the POOL must be BUILT with, derived from the permanent filter. + + ⛔ THIS EXISTS BECAUSE `team_id` SHAPES VALUES, NOT ROW MEMBERSHIP (C-PERM amendment 3). + `modules/customer_data._pool_build` passes `team_id` into `cust._cust_rev` three times (YTD, + LY, LTM) and into `cust._cadence_bulk`, so it decides what `rev`, `ly`, `ltm`, `aov`, + `est_missed` and the derived `status` MEAN. Enforce a BU purely as a post-filter and a + Fisch-only user keeps a correct-looking row LIST while every number on it silently becomes + Fisch+Royal — worst for `dba = Both` customers, who are exactly the ones a BU filter admits. + A pid-level reconciliation cannot see that; the values one can, and does. + + So the query-level pushdown SURVIVES — but as a DERIVATION OF the permanent filter rather + than a second wall beside it, which is what keeps R1's "BU access is just a filter" true at + the level the owner asked for it (one declaration, one UI, one engine). + + Pure, and recomputed per request rather than stored: a stored derivation drifts from the + filter it came from, and then two things disagree about what an account may see. + + Reads TOP-LEVEL AND-conjunction leaves ONLY. A leaf under `or` guarantees no narrowing — + `dba is Fisch OR revenue > 10` must not pin the pool to Fisch — so it never pushes down. + Anything not recognised here simply is not pushed down; `apply_row_scope` still applies the + whole tree, so the wall is unchanged either way. Belt AND braces, deliberately: the pushdown + is what makes the VALUES right, `permits()` is what makes the ROWS right. + + ⭐ THE `bus` FALLBACK, AND WHY IT IS NOT A HOLE IN AMENDMENT 4. On a topic outside + `BU_FILTERABLE_MODULES` there is no column a BU condition could be written against, so + "the filter pins no team" cannot mean "the admin chose consolidated" — it is the only answer + the filter language has. Resolving that silence as None returns the WIDER scope, which makes + the current code fail-OPEN on the values axis for exactly the topic that cannot argue back. + So the record's own `bus` answers instead, and the direction is what makes it safe: this can + only ever REPLACE None (both units) with a pinned single unit. It never widens, it never + touches `may_access`, and a `bus:'all'` account is unaffected because `scope_team_id` returns + None for it — which is every account in tenant #0's registry except the one this shipped for. + """ + if perms.is_admin(user): + return None, None + team_id, agent = _derive_from_filter(user, module) + if team_id is None and module not in BU_FILTERABLE_MODULES: + team_id = perms.scope_team_id(user) + return team_id, agent + + +def _derive_from_filter(user, module): + """`(team_id, agent)` the PERMANENT FILTER pins, before any fallback. Split out so the + fallback has exactly one place to apply — the three exits below all mean "the filter pinned + nothing", and a rule written at each of them is a rule that will one day be written at two.""" + e = entry(user, module) + tree = (e or {}).get('filter') + if not tree: + # Un-migrated records still answer through the legacy derivation, so the old wall keeps + # working until the migration has run. + if not is_migrated(user): + return perms.scope_team_id(user), perms.scope_agent(user) + return None, None + + from harness import filter_eval as fe + nodes, conj = fe.tree_parts(tree) + if conj == 'or': + return None, None + + team_id, agent = None, None + for n in nodes: + if not isinstance(n, dict): + continue + if isinstance(n.get('children'), list): + # A top-level OR GROUP under an AND root IS a guaranteed narrowing — every row must + # satisfy it — so it may push down, unlike a leaf under an OR ROOT (refused above). + # This is the shape a multi-value BU condition actually takes; see `_group_dba_team`. + tid = _group_dba_team(n) + if tid is not None: + team_id = tid if team_id in (None, tid) else None + continue + if n.get('op') not in _PUSHDOWN_OPS: + continue + col = n.get('colId') + raw = n.get('value') + if col == 'dba': + vals = {v.strip().lower() for v in str(raw or '').split(',') if v.strip()} + if not vals: + continue + for allowed, tid in _DBA_TEAM: + if vals <= allowed: + # Both BUs named = no narrowing to push; leave it to the post-filter. + team_id = tid if team_id in (None, tid) else None + break + elif col == 'agent': + v = str(raw or '').strip() + # A SET of agents cannot become the pool's single `agent_name`; the post-filter + # handles it. Only an unambiguous single value pushes down. + if v and ',' not in v: + agent = v + return team_id, agent + + +def _group_dba_team(group): + """The team a top-level `or` group pins, or None. + + Recognises ONLY the exact shape "every child is a `dba eq ` leaf" — the group the + condition builder emits for a multi-value BU condition, and the one `perm_migrate` writes. + Every OTHER group returns None and is left entirely to the post-filter: a group mixing `dba` + with another column, or containing a nested group, does not pin a BU on its own, and + guessing that it does would build the pool from the wrong book. Narrow by construction — + the pushdown may only ever be an OPTIMISATION of a constraint the filter already expresses. + """ + if group.get('conj') != 'or': + return None + children = group.get('children') or [] + if not children: + return None + vals = set() + for c in children: + if (not isinstance(c, dict) or isinstance(c.get('children'), list) + or c.get('colId') != 'dba' or c.get('op') != 'eq'): + return None + v = str(c.get('value') or '').strip().lower() + if not v: + return None + vals.add(v) + for allowed, tid in _DBA_TEAM: + if vals <= allowed: + return tid + return None + + +# ── C1: THE ONE DOOR TO ANY DATABASE'S ROWS (wave 36, W36-T20) ──────────────────────────────── +#: ⭐⭐ OWNER RULING R6, AND IT IS WHY THIS SECTION EXISTS AT ALL: *"EVERY database gets the same +#: permission logic, always"* — per-user field visibility AND row filtration on every database +#: carrying a unique id, whatever created it, with a NEW database inheriting it by construction +#: rather than by a list somebody maintains. +#: +#: ⛔ THE PRODUCT HAD TWO PERMISSION SYSTEMS AND ONLY ONE WAS ARMED. Everything above this line +#: walls the REGISTRY topics (`customer_data`, `product_data`) and is called only from the topic +#: assemblies. Every OTHER database is a `ut_*` table walled by `user_tables.may_open` alone — +#: creator, admin, or a `core.shares` grant — which is a BINARY door: you see all 31,418 rows of +#: `ut_odoo_invoices` or none of them. `perms.tenant_governable_modules`' docstring booked this +#: work in as many words (*"Arming `perm_scope` over `ut_*` … booked, not faked"*), and owner +#: item 11 is that booking coming due. +#: +#: ⚠ AND THE PREMISE THE GRILL GOT WRONG, because the fix depends on it: those databases are NOT +#: user-created. Ten of them (`ut_odoo_invoices`, `…_orders`, `…_agents`, `…_accounts`, `…_bills`, +#: `…_vendors`, `…_order_lines`, `…_gl_lines`, `…_customers`, `…_products`) are generated by the +#: KEYCHAIN connector (`aios-web/api/odoo_relational.py`). **`ut_` is a storage prefix, not a +#: statement about origin**, and a wall keyed off it was reading a naming artefact as a security +#: boundary. +#: +#: ⛔⛔ THE TWO QUESTIONS STAY TWO QUESTIONS. `may_open` answers *"IF you see this database"* and +#: is untouched by this section; C1 answers *"WHICH rows and fields"*. `may_read` below COMPOSES +#: them — it calls `may_open`, it does not reimplement it — because merging them is how this +#: codebase got two ideas of who owns a table once already (`user_tables.may_open`'s own wave-20 +#: note). One resolver per question, asked in order. + + +class UnknownTable(LookupError): + """No database in this tenant answers to that key. + + ⛔ RAISED, NEVER RETURNED AS AN EMPTY LIST (contract C1). An empty list reads as *"this + database is empty"* — indistinguishable from a real empty table, and the caller least able to + notice is the one that wanted rows. This repo has shipped that exact silent-empty answer + before (`user_tables.all_defs`' own correction note; [[empty-answer-vs-unfinished-answer]]). + """ + + +class Denied(PermissionError): + """This principal may not read this database at all. The IF question, answered by `may_read`.""" + + +class Unresolvable(RuntimeError): + """The rows exist and cannot be served under this call's constraints — R6's SECOND SENTENCE. + + ⭐ STANDING RULE 1 IS TWO SENTENCES AND THE SECOND IS THE HALF THAT GETS DROPPED: *"if there + is lag or it can't be done, you need to explicitly tell me why and recommend a fix"*. So a + limit that genuinely cannot be removed is REPORTED with its cause and a recommendation, never + silently enforced as a short answer. Carries the same four keys + `routes_tables._PID_SCOPE_LIMIT` already puts on the wire, so a route can hand this straight + to a client without a second vocabulary ([[one-question-two-normalizers]]). + """ + + def __init__(self, subject, effect, cause, recommendation): + self.subject, self.effect = subject, effect + self.cause, self.recommendation = cause, recommendation + super().__init__(f"{subject}: {effect}. {cause}. {recommendation}") + + def as_limit(self): + """The dict shape `routes_tables` puts in an assembly's `limits` list.""" + return {"subject": self.subject, "effect": self.effect, + "cause": self.cause, "recommendation": self.recommendation} + + +#: Row readers DECLARED by the app layer, keyed by EXACT database key. +#: `reader(table_key, user, st) -> (fields, rows)`. +#: +#: ⛔ WHY A REGISTRY AND NOT AN IMPORT. `core` never imports up (`platform/ARCHITECTURE.md`), and +#: a registry TOPIC's rows are built by `modules/` + `aios_grid` behind an API-layer pool cache +#: (`routes_customers._pool_for`), which is two layers above this file. Same idiom `user_tables` +#: already uses for exactly this reason — `register_connected`, `register_read_through`, +#: `ROW_HOOKS`: *"`core` never imports up, so the app tells this layer rather than being +#: interrogated by it."* +_ROW_SOURCES = {} + +#: THE reader for a read-through `ut_*` grid — one reader, because there is one mirror. +#: `reader(table_key, field_keys, st) -> rows`. +_MIRROR_READER = None + + +def register_rows(reader, *table_keys): + """Declare who reads a NAMED database's rows. Returns the registered key set. + + ⚠ The return value is the registrar's own answer on purpose: a public function whose only + caller is a `verify_*.py` file is a feature no user can reach, and this repo has a gate that + says so ([[reachable-is-not-the-same-as-built]]). Routing the read door through the write + door's return keeps one construction site of the set instead of two. + """ + for key in table_keys: + k = str(key or '').strip() + if k: + _ROW_SOURCES[k] = reader + return frozenset(_ROW_SOURCES) + + +def register_mirror(reader): + """Declare THE reader for read-through `ut_*` grids (`routes_tables._read_through_rows`).""" + global _MIRROR_READER + _MIRROR_READER = reader + return _MIRROR_READER is not None + + +#: ⛔ `row_sources()` IS DELETED (W36-T24 / owner item 13), AND THE REASON IS THE ONE THIS WAVE +#: KEEPS FINDING. It returned `frozenset(_ROW_SOURCES)` under a docstring calling itself *"The ONE +#: list to read"* — and `register_rows` ALREADY returns exactly that, which is the same idiom +#: `user_tables.register_connected_prefix` uses and the same reason: routing the read door through +#: the write door's return keeps ONE construction site of the set. A second accessor beside it is a +#: parallel path with nothing of its own to say, and it shipped with no caller outside `verify_*.py` +#: — the shape that is whole, correct and unreachable ([[artifact-with-no-importer]]; reported by +#: the integrator's `web_reachability` pass, `mailbox/A.md` A-43). The registrar's return is the +#: read: `routes_grid._C1_ROW_SOURCES` is that value, held where it is registered. + + +def _ut(): + import core.user_tables as user_tables + return user_tables + + +def may_read(user, table_key, st=None): + """May this principal read `table_key` AT ALL — the IF question, on EVERY database. + + ⛔ COMPOSED, NOT RE-DERIVED, and the order is the whole rule: + + 1. an admin reads everything (break-glass — `deps._user_for` hands back a hardcoded master + dict on a store outage and it will never carry a `perms` block); + 2. an EXPLICIT stored `access: false` DENIES, on any database. This is the toggle owner + item 11 asks for, and it is a **deny-only overlay**: it may revoke a database the wall + below would admit, and it may never grant one that wall refuses; + 3. a `ut_*` database defers to `user_tables.may_open` — creator, admin, or a `core.shares` + grant — UNMODIFIED. W36-T21: *"`may_open` still decides IF the database is visible."* + 4. anything else is a registry topic and defers to `may_access` above. + + ⛔⛔ WHY ABSENCE MUST NOT DENY ON A `ut_*` KEY, which is the opposite of what leg 4 does. + `may_access` reads migrated-and-undeclared as DENY — correct for a topic, because + `routes_admin` writes an entry for every governable topic on every save. ⚠ NO `ut_*` ENTRY WAS + STORABLE AT ALL UNTIL W36-T22 — `_clean_perms` refused the key with `unenforced_module` — so + every record migrated before this wave carries no entry for any of them, and reading that + absence as a decision would revoke all ten keychain databases from every migrated account the + moment this arms. That is not R6, it is an outage. Leg 3 therefore asks the wall that HAS been + answering rather than the marker that has not, and it keeps being right AFTER the flag's + deletion: an admin who has never opened the editor for a database has still not decided + anything about it. + + ⛔⛔ PASS THE **PUBLIC** RECORD, NOT THE ONE OUT OF `users.json`. Leg 3 needs a username, and a + stored record is keyed BY username in that bucket and does not carry one INSIDE it — only + `core.users._public(uname, rec)` puts it there, which is what `deps.Session.user` holds. Hand + this the raw record and `may_open` gets a `None` viewer and fail-closes, so every `ut_*` + database reads as DENIED for an account that can open all of them. It fails in the SAFE + direction and is silently wrong, which is the worst pair to debug — it cost two call sites in + one afternoon: a gate double, and `routes_admin.get_perms`' own fix for this very outage. + """ + if perms.is_admin(user): + return True + e = entry(user, table_key) + if e is not None and not bool(e.get('access', True)): + return False + key = str(table_key or '') + if key.startswith(_ut().KEY_PREFIX): + return bool(_ut().may_open(key, (user or {}).get('username'), False, st=st)) + return may_access(user, table_key) + + +def wall_declared(user, table_key): + """Is a ROW or FIELD narrowing declared for this principal on this database? + + ⛔ THE QUESTION A DOOR ASKS BEFORE SERVING ROWS IT CANNOT SCOPE. `perms.py` warned that a + stored `ut_*` wall would be INERT — *"the editor would say DENY, the table routes would keep + serving, and nothing anywhere would say so"*. A route that cannot apply C1 must therefore + REFUSE for a principal this returns True for, rather than serve the whole database. False for + an admin (they bypass the wall entirely) and for any record with no entry, so a door asking + this pays nothing and changes nothing for everybody who has no wall. + """ + if perms.is_admin(user): + return False + e = entry(user, table_key) + if not e: + return False + return bool(e.get('filter')) or bool(e.get('hiddenFields')) + + +def row_scope_applies(user, table_key): + """Does a permanent ROW filter narrow this principal on this database? + + ⚠ `wall_declared`'s narrower half, and it exists so a rows-free caller can SKIP building rows + it would only need in order to filter them. `routes_tables.scoped_pids` is that caller: its + whole point is that the pid set costs no row pass, and paying for one on every database + switch — for every account, walled or not — would undo W30-T30 to enforce a rule that applies + to almost nobody. Asked here rather than spelled out at the call site, so there is ONE + statement of when the row wall bites ([[one-question-two-normalizers]]). + """ + if perms.is_admin(user): + return False + return bool((entry(user, table_key) or {}).get('filter')) + + +def scoped_table(user, table_key, st=None, ctx=None): + """⭐⭐ CONTRACT C1 — the rows of ANY database, already field-stripped and row-filtered for + `user`. Registry topic or `ut_*`; there is no third kind and no per-database branch. + + rows = scoped_table(user, 'ut_odoo_invoices') # a keychain database + rows = scoped_table(user, 'customer_data') # a registry topic + + `user` is a user RECORD (the dict `deps.Session.user` carries), not a username — the whole + wall is a pure function of that record. BOTH arguments are positional and REQUIRED: a caller + that forgets the principal must not run, because the only thing a defaulted one could mean is + "unscoped", which is the widening direction. + + ⛔ FAIL-CLOSED, THREE WAYS, AND EACH IS A DIFFERENT EXCEPTION so a caller can answer with the + right status instead of guessing: `UnknownTable` (no such database — never an empty list), + `Denied` (the IF question said no), `Unresolvable` (the rows cannot be served and here is + why — standing rule 1's second sentence). + + ⚠ NO CAP. A connected source is read THROUGH the mirror in full (standing rule 1); the only + thing that stops it is a population that exceeds one materialisation window, and that arrives + as `Unresolvable` carrying its cause and a recommendation rather than as a short answer. + + ⭐ E's SCRIPT SANDBOX HOLDS NO SECOND PATH TO THE STORE (wiring W1), which is why this is + THE door rather than A door: everything a sandboxed script may read, it reads here, under the + CALLING user's scope (R5). + """ + _fields, rows = _scoped(user, table_key, st=st, ctx=ctx) + return rows + + +def scoped_fields(user, table_key, st=None): + """The COLUMNS of any database this principal may see — C1's other half. + + ⛔ IT IS NOT A CONVENIENCE, IT IS THE SECOND WIRE. `strip_row`'s own note above says it: the + field list and the row payload are two different wires, and narrowing one without the other + leaves the value sitting where anything can read it. A caller that must render a scoped table + needs both, and E cannot read a `ut_*` definition to learn its columns — the sandbox has no + second path to the store (W1). So both come from here, off one wall. + + ⚠ On a `ut_*` database this reads the DEFINITION only — the projection, no rows (D-213). On a + registry topic it goes through the registered reader, which builds that topic's pool; the + pool is cached per scope on the tenant runtime, so it is a cache hit next to `scoped_table`. + """ + fields, _rows = _scoped(user, table_key, st=st, ctx=None, want_rows=False) + return fields + + +def _scoped(user, table_key, st=None, ctx=None, want_rows=True): + """`(fields, rows)` — ONE evaluator behind both public doors, so they cannot disagree.""" + key = str(table_key or '').strip() + if not key: + raise UnknownTable('a database key is required. This door will not guess which database ' + 'was meant') + if not may_read(user, key, st=st): + raise Denied(f"this account may not read '{key}'") + fields, rows = _read(key, user, st, want_rows) + # THE FIELD WALL — a TRANSITIVE closure, so hiding a column also hides every formula computed + # FROM it. Resolved ONCE and used for both wires; see `hidden_keys` for why a set difference + # is the wrong shape here. + # ⭐ W38-T16 — `st` RIDES INTO THE WALL, not just into the read. The field-grant leg resolves + # against `object_shares` in THIS tenant's store; without the handle it would answer from the + # module default (tenant #0) and hide a grantee's own column on every other tenant. + hide = hidden_keys(user, key, fields, st=st) + if not want_rows: + return (visible_fields(fields, user, key, st=st) if hide else fields), [] + # THE ROW WALL — `permits()`, so a permanent filter this evaluator cannot answer DENIES + # rather than being ignored. Evaluated against the UNSTRIPPED contract on purpose: a + # permanent filter may name a column the reader is not allowed to SEE, and dropping the + # predicate would widen the read rather than narrow it. + rows = apply_row_scope(rows, user, key, fields, ctx) + if hide: + fields = visible_fields(fields, user, key, st=st) + rows = [strip_row(r, hide) for r in rows] + return fields, rows + + +def _read(table_key, user, st, want_rows=True): + """`(fields, rows)` BEFORE the wall — the app layer's reader, or core's own for a `ut_*`.""" + reader = _ROW_SOURCES.get(table_key) + if reader is not None: + fields, rows = reader(table_key, user, st) + return list(fields or ()), list(rows or ()) + ut = _ut() + if not table_key.startswith(ut.KEY_PREFIX): + # ⛔ A TOPIC WITH NO REGISTERED READER IS UNKNOWN, NOT EMPTY. In a process that never + # imported the API layer this is the honest answer: nothing here can build that pool. + raise UnknownTable(f"no database named '{table_key}' in this workspace, and no reader " + f"is registered for it") + return _read_user_table(table_key, st, want_rows) + + +def _read_user_table(table_key, st, want_rows=True): + """core's OWN reader for a `ut_*` database. Answers with NO registrar, deliberately. + + ⭐ WHY IT LIVES IN `core` RATHER THAN BEING REGISTERED LIKE THE TOPICS, and it is the same + argument that seeds `user_tables._CONNECTED_PREFIXES` rather than registering it: a cold + process — E's sandbox subprocess, a worker, a gate — that never imported an API route still + owes the right answer for `ut_odoo_invoices`. A registrar-only design would raise there, and + the sandbox is exactly such a process. + + ⚠ THE WALL IS ANSWERED ON A PROJECTION AND THE ROWS ARE NOT. `lend_defs` serves definitions + without the 28.6 MB of rows (D-213), which is every read this function makes when + `want_rows` is false; a projected document RAISES on `rows` rather than answering empty, so + the materialised arm below takes the whole read explicitly. + """ + ut = _ut() + lent = ut.lend_defs(st) + defn = ut.get(table_key, st=lent) + if not defn: + raise UnknownTable(f"no database named '{table_key}'") + fields = [dict(f) for f in (defn.get('fields') or [])] + if not want_rows: + return fields, [] + if not ut.materialises(table_key, st=st, defn=defn): + # A read-through grid stores no rows here — they live in the mirror, and reading + # `defn['rows']` would find an empty dict and serve an EMPTY GRID: correct-looking, + # wrong, and silent. + if _MIRROR_READER is None: + raise Unresolvable( + subject='rows', effect='unreadable', + cause=(f"'{table_key}' is served read-through from the connector mirror and no " + f'mirror reader is registered in this process'), + recommendation=('call `perm_scope.register_mirror(...)` from the app layer before ' + 'reading a read-through database, or read it through the API')) + keys = {f['key'] for f in fields if f.get('key')} + return fields, list(_MIRROR_READER(table_key, keys, st) or ()) + whole = ut.get(table_key, st=st) + if whole is None: + # Deleted between the wall and here. The same refusal, not an empty table. + raise UnknownTable(f"no database named '{table_key}'") + field_keys = {f['key'] for f in fields if f.get('key')} + rows = [] + for rid, row in (whole.get('rows') or {}).items(): + if not str(rid).isdigit(): + continue + r = {k: v for k, v in (row or {}).items() if k in field_keys} + r['pid'] = int(rid) + rows.append(r) + rows.sort(key=lambda r: r['pid']) + return fields, rows diff --git a/platform/core/script_sandbox.py b/platform/core/script_sandbox.py index 231ce07dd952f2d67caf31e11e9f1525a6a5a38f..babf60594d17d540e348efa836e40056260a1f96 100644 --- a/platform/core/script_sandbox.py +++ b/platform/core/script_sandbox.py @@ -1,519 +1,519 @@ -"""core/script_sandbox.py — WAVE 36 (R5 / R10, contract C1): running a tenant's OWN Python. - -Owner item 6: *"Add code script as an interface (database View) so a user can build whatever they -want through the Agent chat interface."* Item 8: *"We need to really guardrail the reach of this -script. So let's really grill this down."* R10 ruled it SERVER-SIDE PYTHON after the trade was -stated, so this file is the guardrail, and one engine serves both items. - -════════════════════════════════════════════════════════════════════════════════════════════════ -⛔⛔ THE ONE PARAGRAPH TO READ BEFORE CHANGING ANYTHING HERE. - -In-process CPython cannot deliver two of this ticket's clauses. An AST allow-list plus a curated -namespace stops import, file, network and environment access — but it **cannot cap memory and -cannot interrupt a runaway loop**, because a `while True:` in the same interpreter is not a slow -request, it is the tenant's ONE FastAPI process gone. So the script runs in a **SUBPROCESS**: -`resource.setrlimit` for address space and CPU, a hard wall-clock kill from the parent, and the -allow-list inside. Neither half is sufficient; both are load-bearing. - -⭐ AND THE SUBPROCESS RECEIVES **ROWS, NEVER A STORE**. The parent calls C1's `scoped_table` under -the CALLING user's record and serialises the result; the child imports nothing from this repo and -holds no credential, no runtime and no store handle. Wiring W1 ("the sandbox has no second store -path") is then true by CONSTRUCTION rather than by discipline, and it is checkable: the child -reports its own `sys.modules`, and no `core.*` name may appear in it. - -⛔ NEVER A BLACKLIST. Every rule below is an ALLOW-LIST — a set of node types, a set of attribute -names, a dict of builtins. A blacklist of dangerous spellings is bypassable by construction, and -the bypass is usually one string method away (`"{0.__class__}".format(x)` performs its attribute -lookup inside `format`, so there is no `ast.Attribute` node to refuse). -════════════════════════════════════════════════════════════════════════════════════════════════ - -The two layers, and they refuse DIFFERENT things on purpose: - - 1. `check_source()` — a pure function over source text. Refuses a construct the language offers - and this sandbox does not: `import`, `class`, `with`, `async`, `yield`, `global`, and every - attribute name outside `ALLOWED_ATTRS`. - 2. `SANDBOX_BUILTINS` — the names that resolve at all. `__import__`, `open`, `eval`, `exec`, - `compile`, `getattr`, `globals`, `vars` and `type` are simply absent, so a source that gets - past layer 1 still finds nothing to call. - -⚠ THAT DUPLICATION IS DELIBERATE AND IT CHANGES HOW THE GATE MUST BE WRITTEN. `import os` is -refused twice, so a negative control that drops ONE layer sees the other refuse and reports -green — the shape that already cost this wave one missed control in `routes_agent_harness`. So -each layer is tested AT ITS OWN BOUNDARY: `check_source()` is called directly on source strings, -and `run()` is driven end to end. An NC drops one entry from one frozenset and the matching -boundary goes red. -""" -import ast -import json -import os -import subprocess -import sys -import tempfile -import time -from pathlib import Path - -#: Wall clock, enforced by the PARENT with a kill. The one cap that works on every platform. -DEFAULT_TIMEOUT_S = 10.0 - -#: Address space for the child (`RLIMIT_AS`). POSIX only — see `run()`'s `caps` report. -DEFAULT_MEMORY_BYTES = 512 * 1024 * 1024 - -#: CPU seconds for the child (`RLIMIT_CPU`). POSIX only. Deliberately above the wall clock: the -#: wall-clock kill is the primary control and this is the backstop for a child that stops being -#: reachable. A CPU limit BELOW the timeout would make every slow script look like a CPU refusal. -DEFAULT_CPU_SECONDS = 15 - -#: What the script may print, in bytes. `print` is a curated builtin writing to a capped buffer, -#: and the child's real stdout goes to DEVNULL — so a script cannot fill a pipe, and anything -#: that escaped far enough to write to fd 1 has nowhere for it to land. -MAX_STDOUT_BYTES = 64 * 1024 - -#: The serialised ROW payload handed to the child. ⛔ A REFUSAL, NEVER A TRUNCATION (standing rule -#: 1): a short answer from a data tool is a wrong answer that looks right. Over this, `run()` -#: returns a named limit carrying its cause and a recommendation. -MAX_PAYLOAD_BYTES = 32 * 1024 * 1024 - -#: The emitted spec. A render spec is a description of a picture; one larger than this is data -#: pretending to be a description. -MAX_SPEC_BYTES = 2 * 1024 * 1024 - -MAX_SOURCE_BYTES = 128 * 1024 - - -# ══════════════════════════════════════════════════════ LAYER 1 — the AST allow-list ═══════════ -#: Every `ast` node class a script may contain. ⛔ THE ABSENCES ARE THE POLICY: `Import` / -#: `ImportFrom` (no module reaches the script), `ClassDef` (a class body is a namespace with its -#: own scoping rules and buys a data script nothing), `With` (a context manager is `__enter__` -#: by another spelling), `Global` / `Nonlocal` (rebinding the sandbox's own names), and every -#: `Async*` / `Await` / `Yield` form (this engine is synchronous; a coroutine that is never -#: awaited is a silent no-op that looks like a working script). -ALLOWED_NODES = frozenset(""" -Module Expr Assign AugAssign AnnAssign NamedExpr Return Pass Break Continue Delete Assert Raise -If For While Try TryStar ExceptHandler FunctionDef Lambda arguments arg keyword -BoolOp BinOp UnaryOp IfExp Dict Set List Tuple Starred Subscript Slice Compare Call Attribute Name -Constant JoinedStr FormattedValue ListComp SetComp DictComp GeneratorExp comprehension -Load Store Del -And Or Not Invert UAdd USub -Add Sub Mult Div FloorDiv Mod Pow LShift RShift BitOr BitXor BitAnd MatMult -Eq NotEq Lt LtE Gt GtE Is IsNot In NotIn -""".split()) - -#: Every attribute name a script may READ or CALL. ⛔⛔ THIS IS THE LOAD-BEARING SET, and it is -#: an allow-list of NAMES rather than a refusal of dunders, because the interesting escapes are -#: ordinary-looking: `f.__globals__` on any function reaches the runner's own module namespace, -#: `e.__traceback__.tb_frame.f_globals` reaches it from an exception handler, and `().__class__` -#: reaches `object.__subclasses__`. None of those names is here, and neither is any name this -#: sandbox has not been asked for. -#: ⚠ `format` IS ABSENT DELIBERATELY. `"{0.__class__}".format(x)` performs the attribute lookup -#: INSIDE `str.format`, where no `ast.Attribute` node exists for layer 1 to see. f-strings are -#: fine — `f"{x.__class__}"` compiles to a real `Attribute` node and is refused. -ALLOWED_ATTRS = frozenset(""" -append extend insert pop remove clear sort reverse copy count index -keys values items get setdefault update -add discard union intersection difference issubset issuperset -join split rsplit splitlines strip lstrip rstrip lower upper title capitalize casefold -replace startswith endswith find rfind zfill ljust rjust center partition removeprefix removesuffix -isdigit isalpha isalnum isspace isupper islower isnumeric -real imag numerator denominator -""".split()) - - -class Refused(Exception): - """A named refusal: `code` for a caller to branch on, `message` for a person to read.""" - - def __init__(self, code, message): - self.code, self.message = code, message - super().__init__(f"{code}: {message}") - - -def _attr_ok(name): - """An attribute name passes only if it is on the list AND is not private. - - ⚠ THE SECOND TEST IS NOT A BLACKLIST — it narrows an allow-list that already excludes every - private name. It is here so that adding a name to `ALLOWED_ATTRS` cannot open a dunder by - accident, which is the one edit a future reader is most likely to make in a hurry. - """ - return name in ALLOWED_ATTRS and not name.startswith("_") - - -def check_source(source): - """LAYER 1. Return a `Refused` for source this sandbox will not run, or `None`. - - ⭐ PURE, AND THAT IS WHAT MAKES IT TESTABLE AT ITS OWN BOUNDARY. It reads no file, spawns no - process and touches no store, so a gate can hand it a hundred hostile strings for free and an - NC can drop one entry from one frozenset and watch exactly this function change its answer. - """ - text = str(source or "") - if len(text.encode("utf-8", "replace")) > MAX_SOURCE_BYTES: - return Refused("source_too_long", - f"a script view is at most {MAX_SOURCE_BYTES // 1024} KB of source") - try: - tree = ast.parse(text) - except SyntaxError as exc: - return Refused("syntax", f"line {exc.lineno or 0}: {exc.msg}") - - for node in ast.walk(tree): - kind = type(node).__name__ - if kind not in ALLOWED_NODES: - return Refused("refused_construct", - f"line {getattr(node, 'lineno', 0)}: this sandbox does not run " - f"{_english(kind)}") - if isinstance(node, ast.Attribute) and not _attr_ok(node.attr): - return Refused("refused_attribute", - f"line {getattr(node, 'lineno', 0)}: the attribute " - f"'{node.attr}' is not available inside a script view") - # ⛔ A NAME may not be private either. `_` prefixed names are the runner's own, and a - # script that could bind one could shadow the machinery it runs on top of. - if isinstance(node, ast.Name) and node.id.startswith("_"): - return Refused("reserved_name", - f"line {getattr(node, 'lineno', 0)}: names starting with an " - f"underscore are reserved by the sandbox") - if isinstance(node, (ast.FunctionDef, ast.arg, ast.ExceptHandler)) and str( - getattr(node, "name", None) or getattr(node, "arg", "") or "").startswith("_"): - return Refused("reserved_name", - f"line {getattr(node, 'lineno', 0)}: names starting with an " - f"underscore are reserved by the sandbox") - if isinstance(node, ast.keyword) and str(node.arg or "").startswith("_"): - return Refused("reserved_name", - f"line {getattr(node, 'lineno', 0)}: keyword arguments starting with " - f"an underscore are reserved by the sandbox") - return None - - -_ENGLISH = { - "Import": "an import", "ImportFrom": "an import", "ClassDef": "a class definition", - "With": "a with block", "AsyncWith": "a with block", "AsyncFor": "an async loop", - "AsyncFunctionDef": "an async function", "Await": "await", "Yield": "yield", - "YieldFrom": "yield from", "Global": "a global statement", "Nonlocal": "a nonlocal statement", - "Match": "a match statement", -} - - -def _english(kind): - return _ENGLISH.get(kind, f"a {kind} expression") - - -# ══════════════════════════════════════════ LAYER 2 — the namespace, and the child program ═════ -#: The builtins a script may reach, BY NAME. Everything else is a `NameError` in the child. -#: ⛔ THE ABSENCES, again, are the policy: `__import__` `open` `eval` `exec` `compile` `input` -#: `getattr` `setattr` `delattr` `globals` `locals` `vars` `dir` `type` `super` `object` `help` -#: `exit` `breakpoint` `memoryview` `id`. Several are harmless on their own; each one is a step -#: on a published escape, and none has ever been asked for by a script that shapes rows. -#: ⚠ THE EXCEPTION CLASSES ARE HERE BECAUSE `try:` IS, and a `try` block whose `except` clause -#: cannot name what it catches is a construct that reads as supported and is not. They are safe -#: for the same reason everything else is: `Exception.__subclasses__` needs an attribute this -#: sandbox does not allow, so a class object in the namespace is a leaf, not a doorway. -SANDBOX_BUILTIN_NAMES = ( - "abs all any bool bytes callable chr dict divmod enumerate filter float frozenset hash hex " - "int isinstance issubclass iter len list map max min next oct ord pow range repr reversed " - "round set slice sorted str sum tuple zip True False None " - "Exception ValueError TypeError KeyError IndexError ZeroDivisionError ArithmeticError " - "AttributeError StopIteration OverflowError" -).split() - -#: The literal program the child runs. It is TEXT rather than a module because the child must -#: import nothing from this repo: a module would be found on `sys.path` and would drag `core` -#: with it, which is exactly the second store path W1 forbids. -#: ⚠ Every name in here is underscore-prefixed and layer 1 refuses a script from binding one, so -#: the runner's own machinery cannot be shadowed by the source it executes. -_RUNNER = r''' -import json as _json, os as _os, sys as _sys - -_pay = _json.loads(open(_sys.argv[1], "r", encoding="utf-8").read()) -_out = {"ok": False, "code": "not_run", "message": "the script did not run", - "stdout": "", "spec": None, "caps": {"wallClock": True, "memory": False, "cpu": False}} - -# ── the caps this platform can actually apply, reported either way (standing rule 1) ────────── -try: - import resource as _res - _mem = int(_pay["memoryBytes"]) - _res.setrlimit(_res.RLIMIT_AS, (_mem, _mem)) - _out["caps"]["memory"] = True - _cpu = int(_pay["cpuSeconds"]) - _res.setrlimit(_res.RLIMIT_CPU, (_cpu, _cpu)) - _out["caps"]["cpu"] = True -except Exception: - # `resource` is POSIX only. The wall-clock kill in the parent still applies, and `caps` says - # which of the three held, never a silent partial. - pass - -_printed = [] -_spent = [0] -_LIMIT = int(_pay["maxStdout"]) - - -def _print(*_a, **_k): - _text = (_k.get("sep") or " ").join(str(_x) for _x in _a) + (_k.get("end") or "\n") - _room = _LIMIT - _spent[0] - if _room > 0: - _printed.append(_text[:_room]) - _spent[0] += len(_text) - - -class _Refusal(Exception): - """The SANDBOX refusing, as distinct from the SCRIPT failing. - - Without its own class these arrive as `ValueError`, indistinguishable from a `ValueError` the - script raised itself, and the answer then says "refused" about an ordinary bug in the tenant's - own code. Two different facts, two different codes. - """ - - -_emitted = [] - - -def _emit(_spec): - if not isinstance(_spec, dict): - raise _Refusal("emit() takes a view spec, which is a dictionary") - if _emitted: - raise _Refusal("emit() was already called; a script view emits exactly one view") - _emitted.append(_spec) - - -_rows = _pay["rows"] -_fields = _pay["fields"] -_bound = _pay["table"] - - -def _scoped_table(_table=None): - if _table is not None and str(_table) != _bound: - raise _Refusal( - "this script view is bound to the database '" + _bound + "' and asked for '" - + str(_table) + "'. A script view reads its own database only") - return [dict(_r) for _r in _rows] - - -def _scoped_fields(): - return [dict(_f) for _f in _fields] - - -_ns = {"__builtins__": {_n: __builtins__[_n] if isinstance(__builtins__, dict) - else getattr(__builtins__, _n) - for _n in _pay["builtins"]}} -_ns["__builtins__"]["print"] = _print -_ns["print"] = _print -_ns["emit"] = _emit -_ns["scoped_table"] = _scoped_table -_ns["scoped_fields"] = _scoped_fields -_ns["table"] = _bound - -try: - exec(compile(_pay["source"], " - - + + + + + + + Loopable + + + +
+
+ + + diff --git a/web/public/sample_customers.json b/web/public/sample_customers.json index a1df92945d76142f88d5e0f6a1696a666bebafe3..3d70964f7782c4c55380f593138cbd7aae8de7a8 100644 --- a/web/public/sample_customers.json +++ b/web/public/sample_customers.json @@ -1,558 +1,558 @@ -{ - "fields": [ - { - "key": "customer", - "label": "Customer", - "type": "text", - "source": "odoo", - "pinned": true, - "default": true, - "description": "The customer's name in Odoo. One row per customer who ordered in the last 24 months." - }, - { - "key": "partner_id", - "label": "Odoo ID", - "type": "int", - "source": "odoo", - "derived": true, - "default": false, - "description": "The Odoo res.partner id — the key every Odoo document joins on. DERIVED: this row's pid IS the partner id, so a stored copy would be a second source." - }, - { - "key": "odoo_status", - "label": "Odoo record", - "type": "status", - "source": "odoo", - "default": false, - "options": [ - "Active", - "Archived" - ], - "description": "Whether this customer still exists in Odoo. Archived means deleted there." - }, - { - "key": "agent", - "label": "Agent", - "type": "text", - "source": "odoo", - "default": true, - "description": "The sales agent who owns this account." - }, - { - "key": "dba", - "label": "DBA", - "type": "select", - "source": "odoo", - "default": false, - "options": [ - "Fisch", - "Royal", - "Both" - ], - "description": "The brand this customer buys from - Fisch, Royal, or both. Amazon-channel orders are not a DBA." - }, - { - "key": "salesperson", - "label": "Salesperson", - "type": "text", - "source": "odoo", - "default": false, - "description": "Who keyed in most of this customer's orders — not the Agent, who owns the account." - }, - { - "key": "street", - "label": "Street", - "type": "text", - "source": "odoo", - "default": false, - "description": "First address line, from res.partner directly - not the geocoder, so a customer the map cannot place still shows its address." - }, - { - "key": "street2", - "label": "Street 2", - "type": "text", - "source": "odoo", - "default": false, - "description": "Second address line (suite, unit, floor) on the customer's Odoo address." - }, - { - "key": "city", - "label": "City", - "type": "text", - "source": "odoo", - "default": true, - "description": "City on the customer's Odoo address." - }, - { - "key": "state", - "label": "State", - "type": "text", - "source": "odoo", - "default": true, - "description": "State or province on the customer's Odoo address." - }, - { - "key": "country", - "label": "Country", - "type": "text", - "source": "odoo", - "default": false, - "description": "Country on the customer's Odoo address." - }, - { - "key": "zip", - "label": "ZIP", - "type": "text", - "source": "odoo", - "default": false, - "description": "Postal code on the customer's Odoo address." - }, - { - "key": "customer_since", - "label": "Customer since", - "type": "date", - "source": "odoo", - "default": false, - "description": "When this customer was first set up in Odoo." - }, - { - "key": "tags", - "label": "Tags", - "type": "text", - "source": "odoo", - "default": false, - "description": "Odoo labels on this customer, comma-separated." - }, - { - "key": "pricelist", - "label": "Price list", - "type": "text", - "source": "odoo", - "default": false, - "description": "The price list this customer buys on." - }, - { - "key": "payment_terms", - "label": "Payment terms", - "type": "text", - "source": "odoo", - "default": false, - "description": "Payment terms on this customer's account — Net 30, for example." - }, - { - "key": "last_order", - "label": "Last order", - "type": "date", - "source": "odoo", - "default": true, - "description": "Date of the most recent confirmed order." - }, - { - "key": "overdue_days", - "label": "Overdue days", - "type": "int", - "source": "odoo", - "default": true, - "description": "How many days late this customer is running against their own usual ordering rhythm." - }, - { - "_note": "filterable:false — DERIVED ANALYTIC: est_missed is min(cycles missed, 3) x AOV, a score we compute rather than an object the business has, so a condition on it would read as a fact about the customer when it is a fact about our arithmetic. It still displays and still sorts. Until wave 6 this flag also covered the frozen-window presets (revenue_ytd / revenue_ly / orders_24m / aov / yoy_pct); those are now DELETED outright under the owner's no-buildable-presets rule — see _comment. est_missed itself STAYS: no creatable measure or formula reproduces the cadence model behind it.", - "key": "est_missed", - "label": "Est. missed $", - "type": "currency", - "source": "odoo", - "default": true, - "agg": "sum", - "filterable": false, - "description": "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed." - }, - { - "_note": "wave 21 R1 — KEY UNCHANGED, LABEL RENAMED. The computation is a DISJOINT split (ar.py credit_exposure): this column is only the not-yet-due residual, its sibling is the past-grace residual, and the two sum to the total. Under the label 'AR open $' the majority-late book read as 'Overdue > Open', which is nonsense in AR vocabulary — 'open' universally means the total. The label now says what the number is; the key stays so saved views and filters keep working.", - "key": "ar_open", - "label": "AR current $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "Invoiced money owed but not yet due (a 5-day grace applies before it counts as overdue)." - }, - { - "key": "ar_overdue", - "label": "AR overdue $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "Invoiced money past due — same basis as the Collections page." - }, - { - "_note": "wave 21 R1 — the TOTAL, added beside the rename above. AR current $ + AR overdue $, i.e. what most people mean by 'open AR'. Composed from the same ar.credit_exposure rows the siblings use, so it is transitively reconciled by ar.validate()'s residual read_group tie — no second oracle.", - "key": "ar_outstanding", - "label": "AR outstanding $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "Total invoiced money owed right now: AR current $ plus AR overdue $." - }, - { - "key": "ar_exposure", - "label": "Credit exposure $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "The most you could be out if they stopped paying today: open, overdue, draft and not-yet-invoiced." - }, - { - "key": "ar_aged_1_30", - "label": "1-30 days $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $." - }, - { - "key": "ar_aged_31_60", - "label": "31-60 days $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $." - }, - { - "key": "ar_aged_61_90", - "label": "61-90 days $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $." - }, - { - "key": "ar_aged_90_plus", - "label": "90+ days $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "Overdue by more than 90 days. The four aging buckets sum to AR overdue $." - }, - { - "key": "days_to_pay", - "label": "Days to pay", - "type": "int", - "source": "odoo", - "default": false, - "description": "Average days to pay an invoice in full. Blank means no fully paid invoice yet." - }, - { - "key": "top_category", - "label": "Top category", - "type": "text", - "source": "odoo", - "default": false, - "description": "The category this customer spent the most on in the last 12 months." - }, - { - "key": "top_category_pct", - "label": "Top category %", - "type": "pct", - "source": "odoo", - "default": false, - "description": "Share of last-12-months spend that went to the top category." - }, - { - "key": "sku_count", - "label": "SKUs bought", - "type": "int", - "source": "odoo", - "default": false, - "description": "Distinct products bought in the last 12 months." - }, - { - "key": "top_sku", - "label": "Top SKU", - "type": "text", - "source": "odoo", - "default": false, - "description": "The product this customer spent the most on in the last 12 months." - }, - { - "key": "days_since", - "label": "Days since order", - "type": "int", - "source": "odoo", - "default": false, - "description": "Days since the last confirmed order." - }, - { - "key": "typical_gap_days", - "label": "Typical gap days", - "type": "int", - "source": "odoo", - "default": false, - "description": "Days this customer usually goes between orders, from their own history." - }, - { - "key": "notes", - "label": "Notes", - "type": "text", - "source": "overlay", - "default": false, - "description": "Your notes on this customer. Saved in this app only, visible only to you." - } - ], - "rows": [ - { - "pid": 101, - "customer": "Poppy Flowers", - "status": "New", - "agent": "Naomi Linnell Rivera", - "city": "Charlottesville", - "state": "Virginia (US)", - "last_order": "2026-07-21", - "est_missed": 0, - "notes": "", - "country": "United States", - "zip": "02720", - "payment_terms": "30 Days", - "pricelist": "Fisch 1 (USD)", - "tags": "Royal", - "customer_since": "2023-01-10", - "salesperson": "Jessica", - "ar_open": 0, - "ar_overdue": 0, - "ar_exposure": 0, - "top_category": "Styrofoam", - "top_category_pct": 0.47, - "sku_count": 93, - "top_sku": "AQUAFOAM FLORAL FOAM BRICK | 48-Piece per Pack", - "days_to_pay": 34, - "_created": "2023-01-15 09:10:00", - "lat": 25.7617, - "lon": -80.1918, - "odoo_status": "Archived", - "dba": "Royal", - "ar_outstanding": 0 - }, - { - "pid": 102, - "customer": "Meadow & Vine Wholesale", - "status": "Growing", - "agent": "Carla Jimenez", - "city": "Portland", - "state": "Oregon (US)", - "last_order": "2026-07-19", - "est_missed": 0, - "notes": "Expanding to a second storefront.", - "country": "United States", - "zip": "77041", - "payment_terms": "Immediate Payment", - "pricelist": "Royal 1 (USD)", - "tags": "Royal, Key account", - "customer_since": "2024-02-11", - "salesperson": "Naomi", - "ar_open": 1240.5, - "ar_overdue": 0, - "ar_exposure": 1740.5, - "top_category": "Ribbon", - "top_category_pct": 0.95, - "sku_count": 4, - "top_sku": "2\" X 24\" X 36\" GREEN STYROFOAM BOARD", - "days_to_pay": null, - "_created": "2023-02-15 09:11:00", - "lat": 27.9506, - "lon": -82.4572, - "odoo_status": "Active", - "dba": "Fisch", - "ar_outstanding": 1240.5 - }, - { - "pid": 103, - "customer": "Bluestem Floral Supply", - "status": "Growing", - "agent": "Naomi Linnell Rivera", - "city": "Kansas City", - "state": "Missouri (US)", - "last_order": "2026-07-17", - "est_missed": 0, - "notes": "", - "country": "United States", - "zip": "11219", - "payment_terms": "60 Days", - "pricelist": "Royal 1 (USD)", - "tags": "Fisch", - "customer_since": "2025-03-12", - "salesperson": "Karen", - "ar_open": 0, - "ar_overdue": 5120.25, - "ar_exposure": 5120.25, - "top_category": "Foams & Finishes", - "top_category_pct": 0.31, - "sku_count": 27, - "top_sku": "SATIN RIBBON 2IN", - "days_to_pay": 61, - "_created": "2023-03-15 09:12:00", - "lat": 28.5384, - "lon": -81.3789, - "odoo_status": "Active", - "dba": "Both", - "ar_outstanding": 5120.25 - }, - { - "pid": 104, - "customer": "Camellia Row Florist", - "status": "Declining", - "agent": "Devon Marsh", - "city": "Savannah", - "state": "Georgia (US)", - "last_order": "2026-05-30", - "est_missed": 41200, - "notes": "Switched some volume to a local grower.", - "country": "United States", - "zip": "07649", - "payment_terms": "30 Days", - "pricelist": "Fisch 1 (USD)", - "tags": "(none)", - "customer_since": "2026-04-13", - "salesperson": "(none)", - "ar_open": 8300, - "ar_overdue": 940, - "ar_exposure": 11440, - "top_category": "All", - "top_category_pct": 1.0, - "sku_count": 1, - "top_sku": "(none)", - "days_to_pay": 12, - "_created": "2023-04-15 09:13:00", - "lat": 30.3322, - "lon": -81.6557, - "odoo_status": "Active", - "dba": "Royal", - "ar_outstanding": 9240 - }, - { - "pid": 105, - "customer": "Harborlight Wholesale Blooms", - "status": "Growing", - "agent": "Carla Jimenez", - "city": "Seattle", - "state": "Washington (US)", - "last_order": "2026-07-22", - "est_missed": 0, - "notes": "Top-10 account.", - "country": "United States", - "zip": "33125", - "payment_terms": "Immediate Payment", - "pricelist": "Royal 1 (USD)", - "tags": "Royal", - "customer_since": "2023-05-14", - "salesperson": "Jessica", - "ar_open": 0, - "ar_overdue": 0, - "ar_exposure": 0, - "top_category": "Styrofoam", - "top_category_pct": 0.47, - "sku_count": 93, - "top_sku": "AQUAFOAM FLORAL FOAM BRICK | 48-Piece per Pack", - "days_to_pay": 34, - "_created": "2023-05-15 09:14:00", - "lat": 26.1224, - "lon": -80.1373, - "odoo_status": "Active", - "dba": "", - "ar_outstanding": 0 - }, - { - "pid": 106, - "customer": "Dogwood & Fern Co.", - "status": "Dormant", - "agent": "Devon Marsh", - "city": "Asheville", - "state": "North Carolina (US)", - "last_order": "2026-02-11", - "est_missed": 52400, - "notes": "No spring order this year.", - "country": "United States", - "zip": "90210", - "payment_terms": "60 Days", - "pricelist": "Royal 1 (USD)", - "tags": "Royal, Key account", - "customer_since": "2024-06-15", - "salesperson": "Naomi", - "ar_open": 1240.5, - "ar_overdue": 0, - "ar_exposure": 1740.5, - "top_category": "Ribbon", - "top_category_pct": 0.95, - "sku_count": 4, - "top_sku": "2\" X 24\" X 36\" GREEN STYROFOAM BOARD", - "days_to_pay": null, - "_created": "2023-06-15 09:15:00", - "lat": 27.3364, - "lon": -82.5307, - "odoo_status": "Active", - "dba": "Fisch", - "ar_outstanding": 1240.5 - }, - { - "pid": 107, - "customer": "Verbena Market Florals", - "status": "Lost", - "agent": "Naomi Linnell Rivera", - "city": "Austin", - "state": "Texas (US)", - "last_order": "2025-11-04", - "est_missed": 78300, - "notes": "Went with a competitor on freight terms.", - "country": "United States", - "zip": "08701", - "payment_terms": "30 Days", - "pricelist": "Fisch 1 (USD)", - "tags": "Fisch", - "customer_since": "2025-07-16", - "salesperson": "Karen", - "ar_open": 0, - "ar_overdue": 5120.25, - "ar_exposure": 5120.25, - "top_category": "Foams & Finishes", - "top_category_pct": 0.31, - "sku_count": 27, - "top_sku": "SATIN RIBBON 2IN", - "days_to_pay": 61, - "_created": "2023-07-15 09:16:00", - "lat": null, - "lon": null, - "odoo_status": "Active", - "dba": "Both", - "ar_outstanding": 5120.25 - }, - { - "pid": 108, - "customer": "Larkspur Lane Supply", - "status": "New", - "agent": "Carla Jimenez", - "city": "Denver", - "state": "Colorado (US)", - "last_order": "2026-07-14", - "est_missed": 0, - "notes": "First order in April.", - "country": "United States", - "zip": "60614", - "payment_terms": "Immediate Payment", - "pricelist": "Royal 1 (USD)", - "tags": "(none)", - "customer_since": "2026-08-17", - "salesperson": "(none)", - "ar_open": 8300, - "ar_overdue": 940, - "ar_exposure": 11440, - "top_category": "All", - "top_category_pct": 1.0, - "sku_count": 1, - "top_sku": "(none)", - "days_to_pay": 12, - "_created": "2023-08-15 09:17:00", - "lat": 33.749, - "lon": -84.388, - "odoo_status": "Active", - "dba": "Royal", - "ar_outstanding": 9240 - } - ] -} +{ + "fields": [ + { + "key": "customer", + "label": "Customer", + "type": "text", + "source": "odoo", + "pinned": true, + "default": true, + "description": "The customer's name in Odoo. One row per customer who ordered in the last 24 months." + }, + { + "key": "partner_id", + "label": "Odoo ID", + "type": "int", + "source": "odoo", + "derived": true, + "default": false, + "description": "The Odoo res.partner id — the key every Odoo document joins on. DERIVED: this row's pid IS the partner id, so a stored copy would be a second source." + }, + { + "key": "odoo_status", + "label": "Odoo record", + "type": "status", + "source": "odoo", + "default": false, + "options": [ + "Active", + "Archived" + ], + "description": "Whether this customer still exists in Odoo. Archived means deleted there." + }, + { + "key": "agent", + "label": "Agent", + "type": "text", + "source": "odoo", + "default": true, + "description": "The sales agent who owns this account." + }, + { + "key": "dba", + "label": "DBA", + "type": "select", + "source": "odoo", + "default": false, + "options": [ + "Fisch", + "Royal", + "Both" + ], + "description": "The brand this customer buys from - Fisch, Royal, or both. Amazon-channel orders are not a DBA." + }, + { + "key": "salesperson", + "label": "Salesperson", + "type": "text", + "source": "odoo", + "default": false, + "description": "Who keyed in most of this customer's orders — not the Agent, who owns the account." + }, + { + "key": "street", + "label": "Street", + "type": "text", + "source": "odoo", + "default": false, + "description": "First address line, from res.partner directly - not the geocoder, so a customer the map cannot place still shows its address." + }, + { + "key": "street2", + "label": "Street 2", + "type": "text", + "source": "odoo", + "default": false, + "description": "Second address line (suite, unit, floor) on the customer's Odoo address." + }, + { + "key": "city", + "label": "City", + "type": "text", + "source": "odoo", + "default": true, + "description": "City on the customer's Odoo address." + }, + { + "key": "state", + "label": "State", + "type": "text", + "source": "odoo", + "default": true, + "description": "State or province on the customer's Odoo address." + }, + { + "key": "country", + "label": "Country", + "type": "text", + "source": "odoo", + "default": false, + "description": "Country on the customer's Odoo address." + }, + { + "key": "zip", + "label": "ZIP", + "type": "text", + "source": "odoo", + "default": false, + "description": "Postal code on the customer's Odoo address." + }, + { + "key": "customer_since", + "label": "Customer since", + "type": "date", + "source": "odoo", + "default": false, + "description": "When this customer was first set up in Odoo." + }, + { + "key": "tags", + "label": "Tags", + "type": "text", + "source": "odoo", + "default": false, + "description": "Odoo labels on this customer, comma-separated." + }, + { + "key": "pricelist", + "label": "Price list", + "type": "text", + "source": "odoo", + "default": false, + "description": "The price list this customer buys on." + }, + { + "key": "payment_terms", + "label": "Payment terms", + "type": "text", + "source": "odoo", + "default": false, + "description": "Payment terms on this customer's account — Net 30, for example." + }, + { + "key": "last_order", + "label": "Last order", + "type": "date", + "source": "odoo", + "default": true, + "description": "Date of the most recent confirmed order." + }, + { + "key": "overdue_days", + "label": "Overdue days", + "type": "int", + "source": "odoo", + "default": true, + "description": "How many days late this customer is running against their own usual ordering rhythm." + }, + { + "_note": "filterable:false — DERIVED ANALYTIC: est_missed is min(cycles missed, 3) x AOV, a score we compute rather than an object the business has, so a condition on it would read as a fact about the customer when it is a fact about our arithmetic. It still displays and still sorts. Until wave 6 this flag also covered the frozen-window presets (revenue_ytd / revenue_ly / orders_24m / aov / yoy_pct); those are now DELETED outright under the owner's no-buildable-presets rule — see _comment. est_missed itself STAYS: no creatable measure or formula reproduces the cadence model behind it.", + "key": "est_missed", + "label": "Est. missed $", + "type": "currency", + "source": "odoo", + "default": true, + "agg": "sum", + "filterable": false, + "description": "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed." + }, + { + "_note": "wave 21 R1 — KEY UNCHANGED, LABEL RENAMED. The computation is a DISJOINT split (ar.py credit_exposure): this column is only the not-yet-due residual, its sibling is the past-grace residual, and the two sum to the total. Under the label 'AR open $' the majority-late book read as 'Overdue > Open', which is nonsense in AR vocabulary — 'open' universally means the total. The label now says what the number is; the key stays so saved views and filters keep working.", + "key": "ar_open", + "label": "AR current $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Invoiced money owed but not yet due (a 5-day grace applies before it counts as overdue)." + }, + { + "key": "ar_overdue", + "label": "AR overdue $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Invoiced money past due — same basis as the Collections page." + }, + { + "_note": "wave 21 R1 — the TOTAL, added beside the rename above. AR current $ + AR overdue $, i.e. what most people mean by 'open AR'. Composed from the same ar.credit_exposure rows the siblings use, so it is transitively reconciled by ar.validate()'s residual read_group tie — no second oracle.", + "key": "ar_outstanding", + "label": "AR outstanding $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Total invoiced money owed right now: AR current $ plus AR overdue $." + }, + { + "key": "ar_exposure", + "label": "Credit exposure $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "The most you could be out if they stopped paying today: open, overdue, draft and not-yet-invoiced." + }, + { + "key": "ar_aged_1_30", + "label": "1-30 days $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $." + }, + { + "key": "ar_aged_31_60", + "label": "31-60 days $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $." + }, + { + "key": "ar_aged_61_90", + "label": "61-90 days $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $." + }, + { + "key": "ar_aged_90_plus", + "label": "90+ days $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Overdue by more than 90 days. The four aging buckets sum to AR overdue $." + }, + { + "key": "days_to_pay", + "label": "Days to pay", + "type": "int", + "source": "odoo", + "default": false, + "description": "Average days to pay an invoice in full. Blank means no fully paid invoice yet." + }, + { + "key": "top_category", + "label": "Top category", + "type": "text", + "source": "odoo", + "default": false, + "description": "The category this customer spent the most on in the last 12 months." + }, + { + "key": "top_category_pct", + "label": "Top category %", + "type": "pct", + "source": "odoo", + "default": false, + "description": "Share of last-12-months spend that went to the top category." + }, + { + "key": "sku_count", + "label": "SKUs bought", + "type": "int", + "source": "odoo", + "default": false, + "description": "Distinct products bought in the last 12 months." + }, + { + "key": "top_sku", + "label": "Top SKU", + "type": "text", + "source": "odoo", + "default": false, + "description": "The product this customer spent the most on in the last 12 months." + }, + { + "key": "days_since", + "label": "Days since order", + "type": "int", + "source": "odoo", + "default": false, + "description": "Days since the last confirmed order." + }, + { + "key": "typical_gap_days", + "label": "Typical gap days", + "type": "int", + "source": "odoo", + "default": false, + "description": "Days this customer usually goes between orders, from their own history." + }, + { + "key": "notes", + "label": "Notes", + "type": "text", + "source": "overlay", + "default": false, + "description": "Your notes on this customer. Saved in this app only, visible only to you." + } + ], + "rows": [ + { + "pid": 101, + "customer": "Poppy Flowers", + "status": "New", + "agent": "Naomi Linnell Rivera", + "city": "Charlottesville", + "state": "Virginia (US)", + "last_order": "2026-07-21", + "est_missed": 0, + "notes": "", + "country": "United States", + "zip": "02720", + "payment_terms": "30 Days", + "pricelist": "Fisch 1 (USD)", + "tags": "Royal", + "customer_since": "2023-01-10", + "salesperson": "Jessica", + "ar_open": 0, + "ar_overdue": 0, + "ar_exposure": 0, + "top_category": "Styrofoam", + "top_category_pct": 0.47, + "sku_count": 93, + "top_sku": "AQUAFOAM FLORAL FOAM BRICK | 48-Piece per Pack", + "days_to_pay": 34, + "_created": "2023-01-15 09:10:00", + "lat": 25.7617, + "lon": -80.1918, + "odoo_status": "Archived", + "dba": "Royal", + "ar_outstanding": 0 + }, + { + "pid": 102, + "customer": "Meadow & Vine Wholesale", + "status": "Growing", + "agent": "Carla Jimenez", + "city": "Portland", + "state": "Oregon (US)", + "last_order": "2026-07-19", + "est_missed": 0, + "notes": "Expanding to a second storefront.", + "country": "United States", + "zip": "77041", + "payment_terms": "Immediate Payment", + "pricelist": "Royal 1 (USD)", + "tags": "Royal, Key account", + "customer_since": "2024-02-11", + "salesperson": "Naomi", + "ar_open": 1240.5, + "ar_overdue": 0, + "ar_exposure": 1740.5, + "top_category": "Ribbon", + "top_category_pct": 0.95, + "sku_count": 4, + "top_sku": "2\" X 24\" X 36\" GREEN STYROFOAM BOARD", + "days_to_pay": null, + "_created": "2023-02-15 09:11:00", + "lat": 27.9506, + "lon": -82.4572, + "odoo_status": "Active", + "dba": "Fisch", + "ar_outstanding": 1240.5 + }, + { + "pid": 103, + "customer": "Bluestem Floral Supply", + "status": "Growing", + "agent": "Naomi Linnell Rivera", + "city": "Kansas City", + "state": "Missouri (US)", + "last_order": "2026-07-17", + "est_missed": 0, + "notes": "", + "country": "United States", + "zip": "11219", + "payment_terms": "60 Days", + "pricelist": "Royal 1 (USD)", + "tags": "Fisch", + "customer_since": "2025-03-12", + "salesperson": "Karen", + "ar_open": 0, + "ar_overdue": 5120.25, + "ar_exposure": 5120.25, + "top_category": "Foams & Finishes", + "top_category_pct": 0.31, + "sku_count": 27, + "top_sku": "SATIN RIBBON 2IN", + "days_to_pay": 61, + "_created": "2023-03-15 09:12:00", + "lat": 28.5384, + "lon": -81.3789, + "odoo_status": "Active", + "dba": "Both", + "ar_outstanding": 5120.25 + }, + { + "pid": 104, + "customer": "Camellia Row Florist", + "status": "Declining", + "agent": "Devon Marsh", + "city": "Savannah", + "state": "Georgia (US)", + "last_order": "2026-05-30", + "est_missed": 41200, + "notes": "Switched some volume to a local grower.", + "country": "United States", + "zip": "07649", + "payment_terms": "30 Days", + "pricelist": "Fisch 1 (USD)", + "tags": "(none)", + "customer_since": "2026-04-13", + "salesperson": "(none)", + "ar_open": 8300, + "ar_overdue": 940, + "ar_exposure": 11440, + "top_category": "All", + "top_category_pct": 1.0, + "sku_count": 1, + "top_sku": "(none)", + "days_to_pay": 12, + "_created": "2023-04-15 09:13:00", + "lat": 30.3322, + "lon": -81.6557, + "odoo_status": "Active", + "dba": "Royal", + "ar_outstanding": 9240 + }, + { + "pid": 105, + "customer": "Harborlight Wholesale Blooms", + "status": "Growing", + "agent": "Carla Jimenez", + "city": "Seattle", + "state": "Washington (US)", + "last_order": "2026-07-22", + "est_missed": 0, + "notes": "Top-10 account.", + "country": "United States", + "zip": "33125", + "payment_terms": "Immediate Payment", + "pricelist": "Royal 1 (USD)", + "tags": "Royal", + "customer_since": "2023-05-14", + "salesperson": "Jessica", + "ar_open": 0, + "ar_overdue": 0, + "ar_exposure": 0, + "top_category": "Styrofoam", + "top_category_pct": 0.47, + "sku_count": 93, + "top_sku": "AQUAFOAM FLORAL FOAM BRICK | 48-Piece per Pack", + "days_to_pay": 34, + "_created": "2023-05-15 09:14:00", + "lat": 26.1224, + "lon": -80.1373, + "odoo_status": "Active", + "dba": "", + "ar_outstanding": 0 + }, + { + "pid": 106, + "customer": "Dogwood & Fern Co.", + "status": "Dormant", + "agent": "Devon Marsh", + "city": "Asheville", + "state": "North Carolina (US)", + "last_order": "2026-02-11", + "est_missed": 52400, + "notes": "No spring order this year.", + "country": "United States", + "zip": "90210", + "payment_terms": "60 Days", + "pricelist": "Royal 1 (USD)", + "tags": "Royal, Key account", + "customer_since": "2024-06-15", + "salesperson": "Naomi", + "ar_open": 1240.5, + "ar_overdue": 0, + "ar_exposure": 1740.5, + "top_category": "Ribbon", + "top_category_pct": 0.95, + "sku_count": 4, + "top_sku": "2\" X 24\" X 36\" GREEN STYROFOAM BOARD", + "days_to_pay": null, + "_created": "2023-06-15 09:15:00", + "lat": 27.3364, + "lon": -82.5307, + "odoo_status": "Active", + "dba": "Fisch", + "ar_outstanding": 1240.5 + }, + { + "pid": 107, + "customer": "Verbena Market Florals", + "status": "Lost", + "agent": "Naomi Linnell Rivera", + "city": "Austin", + "state": "Texas (US)", + "last_order": "2025-11-04", + "est_missed": 78300, + "notes": "Went with a competitor on freight terms.", + "country": "United States", + "zip": "08701", + "payment_terms": "30 Days", + "pricelist": "Fisch 1 (USD)", + "tags": "Fisch", + "customer_since": "2025-07-16", + "salesperson": "Karen", + "ar_open": 0, + "ar_overdue": 5120.25, + "ar_exposure": 5120.25, + "top_category": "Foams & Finishes", + "top_category_pct": 0.31, + "sku_count": 27, + "top_sku": "SATIN RIBBON 2IN", + "days_to_pay": 61, + "_created": "2023-07-15 09:16:00", + "lat": null, + "lon": null, + "odoo_status": "Active", + "dba": "Both", + "ar_outstanding": 5120.25 + }, + { + "pid": 108, + "customer": "Larkspur Lane Supply", + "status": "New", + "agent": "Carla Jimenez", + "city": "Denver", + "state": "Colorado (US)", + "last_order": "2026-07-14", + "est_missed": 0, + "notes": "First order in April.", + "country": "United States", + "zip": "60614", + "payment_terms": "Immediate Payment", + "pricelist": "Royal 1 (USD)", + "tags": "(none)", + "customer_since": "2026-08-17", + "salesperson": "(none)", + "ar_open": 8300, + "ar_overdue": 940, + "ar_exposure": 11440, + "top_category": "All", + "top_category_pct": 1.0, + "sku_count": 1, + "top_sku": "(none)", + "days_to_pay": 12, + "_created": "2023-08-15 09:17:00", + "lat": 33.749, + "lon": -84.388, + "odoo_status": "Active", + "dba": "Royal", + "ar_outstanding": 9240 + } + ] +} diff --git a/web/src/App.tsx b/web/src/App.tsx index 8e740b30aa739af89fe5978458176d8cc964a6e6..0dcffc3f4790d9b8a0842285b64f4ff08b9d5da4 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -1,41 +1,41 @@ -import { Suspense, lazy } from "react"; -// ⭐⭐ WAVE 37 · T48 — LAZY, AND IT IS THE LAST OF THE THREE EAGER IMPORTS. `CustomerGrid` (7,715 -// lines) plus glide-data-grid sat in the ENTRY chunk because these three files imported it at the -// top level, while twelve other surfaces were already `lazy()`. Measured on the deployed build -// (`proto/P4-first-load.md` §A.2): the entry chunk was 580,102 B on the wire and took 5.6-5.8 s, -// during which `#root` was EMPTY and the screen was white — 6.0-8.0 s of the 11.6-16.0 s first open. -// ⚠ THE EMBED PAYS THIS TOO, and it is the one path where the grid IS the page, so its fallback is -// deliberately the same skeleton shape the shell uses rather than `null`: a blank iframe is -// indistinguishable from a broken one. -const CustomerGrid = lazy(() => import("./customer-grid/CustomerGrid")); -import { OverlayProvider } from "./customer-grid/OverlaySurface"; -import { isStreamlitComponent } from "./customer-grid/hostBridge"; -import Shell from "./shell/Shell"; - -// HOSTED = either Streamlit path (the component iframe, or the legacy injected -// payload). There the surrounding app owns all chrome and this tree renders the -// BARE grid, byte-for-byte the behavior the embed always had. Only the -// standalone build gets the shell (wave 4, the strangler frame). Computed once -// at module level, like useCustomerData's own mode constants — the host mode -// cannot change within a page lifetime. -// EXPORTED (EXIT wave 1): main.tsx needs the same answer to decide whether to -// install the standalone event sink, and two copies of a mode predicate is how -// the two halves of "standalone-only" end up disagreeing. -export const HOSTED = - isStreamlitComponent() || - typeof (window as unknown as { __AIOS_GRID__?: unknown }).__AIOS_GRID__ !== "undefined"; - -export default function App() { - if (HOSTED) { - return ( -
- - - -
- ); - } - return ; -} +import { Suspense, lazy } from "react"; +// ⭐⭐ WAVE 37 · T48 — LAZY, AND IT IS THE LAST OF THE THREE EAGER IMPORTS. `CustomerGrid` (7,715 +// lines) plus glide-data-grid sat in the ENTRY chunk because these three files imported it at the +// top level, while twelve other surfaces were already `lazy()`. Measured on the deployed build +// (`proto/P4-first-load.md` §A.2): the entry chunk was 580,102 B on the wire and took 5.6-5.8 s, +// during which `#root` was EMPTY and the screen was white — 6.0-8.0 s of the 11.6-16.0 s first open. +// ⚠ THE EMBED PAYS THIS TOO, and it is the one path where the grid IS the page, so its fallback is +// deliberately the same skeleton shape the shell uses rather than `null`: a blank iframe is +// indistinguishable from a broken one. +const CustomerGrid = lazy(() => import("./customer-grid/CustomerGrid")); +import { OverlayProvider } from "./customer-grid/OverlaySurface"; +import { isStreamlitComponent } from "./customer-grid/hostBridge"; +import Shell from "./shell/Shell"; + +// HOSTED = either Streamlit path (the component iframe, or the legacy injected +// payload). There the surrounding app owns all chrome and this tree renders the +// BARE grid, byte-for-byte the behavior the embed always had. Only the +// standalone build gets the shell (wave 4, the strangler frame). Computed once +// at module level, like useCustomerData's own mode constants — the host mode +// cannot change within a page lifetime. +// EXPORTED (EXIT wave 1): main.tsx needs the same answer to decide whether to +// install the standalone event sink, and two copies of a mode predicate is how +// the two halves of "standalone-only" end up disagreeing. +export const HOSTED = + isStreamlitComponent() || + typeof (window as unknown as { __AIOS_GRID__?: unknown }).__AIOS_GRID__ !== "undefined"; + +export default function App() { + if (HOSTED) { + return ( +
+ + + +
+ ); + } + return ; +} diff --git a/web/src/account/SubscriptionPage.tsx b/web/src/account/SubscriptionPage.tsx index 59f00a708e4ab698bd6e9863238898e970f0a56c..2cbc71a05dca231d01b3badbb396d5bd70452472 100644 --- a/web/src/account/SubscriptionPage.tsx +++ b/web/src/account/SubscriptionPage.tsx @@ -1,49 +1,49 @@ -// --------------------------------------------------------------------------- -// account/SubscriptionPage.tsx — WAVE 35 · W35-T20 (owner item 13, contract C6). -// -// Owner item 13, verbatim: *"add below settings the following button: Feedback …, -// Usage credits …, and a Subscription module … Specialized for the Royal Imports -// tenant, do not show it."* -// ⛔ "verbatim" IS THE POINT: W37-T02 renamed that surface to "Usage" and this quote KEEPS the -// owner's own wording. A quote edited to match a later rename is no longer evidence of anything. -// -// This is the third of those three, and the only one with nothing behind it yet. -// So it says so, in the words this app already uses for a boarded door -// (`Shell.tsx`'s templates note: "Under construction. … will open here in a later -// release."). One voice for one state — DESIGN.md 1's "variety is a defect". -// -// ⛔ THIS PAGE DOES NOT HIDE ITSELF, AND THAT IS THE TICKET'S OWN TRAP. -// C6 puts the Royal Imports rule in TWO places that are both the frame's: the -// account MENU does not draw the row, and the ROUTE refuses. A third opinion here -// would be a third thing to keep in step, and the day they disagree the reader -// gets a menu row that opens a page which says it does not exist. The page renders -// the same for every tenant that can reach it, because reaching it IS the decision. -// -// ⛔ AND IT TAKES NO PROPS, WHICH IS ALSO A DECISION rather than an omission. -// The wave's rule is that a prop must be REQUIRED, never optional — an optional one -// degrades to "the page does not exist", which is indistinguishable from never -// built. That rule constrains the props that EXIST; it does not ask for a prop that -// carries nothing. There is nothing this page needs from the frame: it holds no -// state, makes no call, and takes no decision the frame has already taken. A -// `tenant` prop here would be exactly the second opinion the paragraph above -// refuses. The mount is proven by A's `verify_wiring` row (W2), not by a signature. -// --------------------------------------------------------------------------- - -import "./account.css"; - -export default function SubscriptionPage() { - return ( -
-

Subscription

-
- {/* ⚠ TWO SHORT SENTENCES AND NO TOUR (DESIGN.md 4 / wave-22 R13: the app does not - narrate itself). The reader arrived here on purpose and wants one fact: is there - anything to do? There is not, and every further word would be spent on someone - who is already leaving. No feature list, no roadmap, no "in the meantime". */} -

- Under construction. Plans, billing and invoices will open here in a later release. -

-
-
- ); -} +// --------------------------------------------------------------------------- +// account/SubscriptionPage.tsx — WAVE 35 · W35-T20 (owner item 13, contract C6). +// +// Owner item 13, verbatim: *"add below settings the following button: Feedback …, +// Usage credits …, and a Subscription module … Specialized for the Royal Imports +// tenant, do not show it."* +// ⛔ "verbatim" IS THE POINT: W37-T02 renamed that surface to "Usage" and this quote KEEPS the +// owner's own wording. A quote edited to match a later rename is no longer evidence of anything. +// +// This is the third of those three, and the only one with nothing behind it yet. +// So it says so, in the words this app already uses for a boarded door +// (`Shell.tsx`'s templates note: "Under construction. … will open here in a later +// release."). One voice for one state — DESIGN.md 1's "variety is a defect". +// +// ⛔ THIS PAGE DOES NOT HIDE ITSELF, AND THAT IS THE TICKET'S OWN TRAP. +// C6 puts the Royal Imports rule in TWO places that are both the frame's: the +// account MENU does not draw the row, and the ROUTE refuses. A third opinion here +// would be a third thing to keep in step, and the day they disagree the reader +// gets a menu row that opens a page which says it does not exist. The page renders +// the same for every tenant that can reach it, because reaching it IS the decision. +// +// ⛔ AND IT TAKES NO PROPS, WHICH IS ALSO A DECISION rather than an omission. +// The wave's rule is that a prop must be REQUIRED, never optional — an optional one +// degrades to "the page does not exist", which is indistinguishable from never +// built. That rule constrains the props that EXIST; it does not ask for a prop that +// carries nothing. There is nothing this page needs from the frame: it holds no +// state, makes no call, and takes no decision the frame has already taken. A +// `tenant` prop here would be exactly the second opinion the paragraph above +// refuses. The mount is proven by A's `verify_wiring` row (W2), not by a signature. +// --------------------------------------------------------------------------- + +import "./account.css"; + +export default function SubscriptionPage() { + return ( +
+

Subscription

+
+ {/* ⚠ TWO SHORT SENTENCES AND NO TOUR (DESIGN.md 4 / wave-22 R13: the app does not + narrate itself). The reader arrived here on purpose and wants one fact: is there + anything to do? There is not, and every further word would be spent on someone + who is already leaving. No feature list, no roadmap, no "in the meantime". */} +

+ Under construction. Plans, billing and invoices will open here in a later release. +

+
+
+ ); +} diff --git a/web/src/account/UsagePage.tsx b/web/src/account/UsagePage.tsx index 6ab2ceb32d8c3e1213c3df4e9667466d4384d916..f735d26732cc9caca5ada0eb71c190c9584ec1b4 100644 --- a/web/src/account/UsagePage.tsx +++ b/web/src/account/UsagePage.tsx @@ -1,297 +1,297 @@ -// --------------------------------------------------------------------------- -// account/UsagePage.tsx — WAVE 35 · W35-T19 (owner item 13, ruling R9, C6/C7). -// -// R9: ONE meter for every AI surface. It REPORTS weekly usage against an -// allowance and does not cut anyone off this wave; over the allowance the -// product still works and the bar is red. -// -// ⛔⛔ THIS PAGE MAY NOT INVENT A FIGURE, AND THAT IS THE WHOLE TICKET. -// Every number here comes off `GET /usage` — the week, the reset date, the -// allowance, each surface's tokens and calls. Nothing is derived from a clock, -// nothing is estimated, and a surface with no ledger lines shows **0**, not a -// blank and not an omission (E-9 guarantees all four surfaces are always -// present, so there is no missing-row branch to get wrong). -// [[no-unverifiable-aggregates]] -// -// ⛔ AND `unmeasured` IS RENDERED, NOT SWALLOWED. Those are real calls whose -// provider declined to report a token count, so the total beside them is a -// FLOOR. A page that hides them presents the floor as a complete figure, which -// is the cost-surprise failure one step removed — the same reason -// `usage_ledger` books an unmeasured call as unknown rather than as zero. -// -// ⚠ SURFACE/READ SPLIT, as on Home and Starred: `renderToStaticMarkup` runs no -// effect, so a shot of the fetching component photographs only its spinner. -// --------------------------------------------------------------------------- - -import { useCallback, useEffect, useRef, useState } from "react"; - -import { fmt } from "../ui/fmt"; -import { loadUsage } from "./accountApi"; -import type { Usage, UsageSurface } from "./accountApi"; -import "./account.css"; - -/** - * The four surfaces C7 names, in the order R9 lists them. - * - * ⚠ AN UNKNOWN KEY IS HUMANISED, NEVER DROPPED. Dropping it would hide real usage from a meter - * whose whole job is to account for it, and printing the raw key would put an internal identifier - * on screen. The day a fifth LLM entry point lands (this product added two in one wave) it shows - * up here as a readable name with the right number beside it, and somebody can then decide what - * to call it. - */ -const SURFACE_LABEL: Record = { - assistant: "Assistant", - field_agent: "Field agents", - ai_review: "Reviews", - automation_draft: "Drafting", -}; -const ORDER = ["assistant", "field_agent", "ai_review", "automation_draft"]; - -export function surfaceLabel(key: string): string { - const known = SURFACE_LABEL[key]; - if (known) return known; - const words = key.replace(/[_-]+/g, " ").trim(); - return words ? words.charAt(0).toUpperCase() + words.slice(1) : key; -} - -/** C7's order first, then anything the server added, alphabetically. */ -export function orderSurfaces(surfaces: UsageSurface[]): UsageSurface[] { - const rank = (s: UsageSurface) => { - const i = ORDER.indexOf(s.surface); - return i === -1 ? ORDER.length : i; - }; - return [...surfaces].sort( - (a, b) => rank(a) - rank(b) || surfaceLabel(a.surface).localeCompare(surfaceLabel(b.surface)) - ); -} - -/** - * The bar's fill, as a percentage, CLAMPED to 100. - * - * ⚠ The clamp is what keeps "over the allowance" a COLOUR rather than a bar that overflows its - * own track and paints across the page. The number above the bar is unclamped and is where the - * overage is actually read. - * ⚠ An allowance of 0 (unset) yields 0 rather than a division by zero: no allowance means the - * meter has nothing to measure against, and a full red bar would be an assertion nobody made. - */ -export function barPct(value: number, of: number): number { - if (!(of > 0)) return 0; - return Math.max(0, Math.min(100, (value / of) * 100)); -} - -export type UsageLoad = - | { phase: "loading" } - | { phase: "ready"; usage: Usage } - | { phase: "failed"; message: string }; - -function Bar({ pct, over }: { pct: number; over: boolean }) { - return ( - {queryCitationLabel(citation)}; -} - -/** One glyph, drawn once and turned over for the other. R20 asked for thumbs; these are thumbs. */ -function ThumbIcon({ down = false }: { down?: boolean }) { - return ( - - ); -} - -export function AssistantMessage({ message, view, citations, onPreview, onRate }: { - message: QueryMessage; view?: SavedQuery; citations: QueryCitation[]; onPreview: (id: string) => void; - /** Optional so the message can be rendered outside a live conversation without a rating door. */ - onRate?: (id: string, rating: "up" | "down" | null, reason?: string) => void; -}) { - const [openSources, setOpenSources] = useState(false); - const [askWhy, setAskWhy] = useState(false); - const [why, setWhy] = useState(""); - const matching = citations.filter((citation) => (message.citationIds || []).includes(citation.id)); - if (message.role === "user") { - return ( -
-

{message.content}

-
- ); - } - const numeric = message.numeric; - return ( -
-

{message.content}

- {numeric && numeric.value !== null && numeric.value !== undefined ? ( -

- {numeric.label} - {numeric.value.toLocaleString()} -

- ) : null} - {view ? ( - - ) : null} - {/* ⚠ The thumbs are NOT a survey: the server reads a thumbs-down back into the next turn of - this thread. That is why a down asks for one short reason and why both are clearable. */} - {onRate ? ( -
- - - {message.rating === "down" && !askWhy && message.ratingReason ? ( - {message.ratingReason} - ) : null} -
- ) : null} - {onRate && askWhy && message.rating === "down" ? ( -
{ - event.preventDefault(); setAskWhy(false); onRate(message.id, "down", why.trim()); - }}> - setWhy(event.currentTarget.value)} /> - -
- ) : null} - {matching.length ? ( -
- - {openSources ? ( -
- {matching.map((citation) => )} -
- ) : null} -
- ) : null} -
- ); -} - -function initialIndex(): QueryIndex { - return { threads: [], messages: [], views: [], citations: [], models: ["auto"], - modelStatus: [], sources: [] }; -} - -/** - * ⭐⭐ OWNER ITEM 2 (2026-08-15) — A PLAIN CHATBOT, IN THREE BORROWED SHAPES. - * - * Verbatim: *"AI assistant module should be a plain chatbot interface where the user can choose - * which model to use… I really like the plain look for Glean… Notice how the database you can - * select is at the bottom there with all the logos, that's how we should allow user to select - * databases too… The interface should exactly be like ChatGPT… ALSO I want a navigation for the - * chat EXACTLY like how Airtable does it."* - * - * So the surface is assembled from the three references the owner supplied, each for the part it - * is actually about: - * · `reference/ChatGPT 1.png` — the CONVERSATION: one centred column, the question as a bubble - * on the right, the answer as plain flowing text with no label and no bubble, the composer - * pinned to the bottom with its controls INSIDE the field. - * · `reference/Glean 1.png` — the SOURCES: a row of chips carrying each source's own mark, - * directly under the composer. Not a dropdown, not a settings panel. - * · `reference/Airtable AI 1.png` — the NAVIGATION: "New chat" at the top of a left panel, the - * thread list under it, and a centred "How can I help?" when the thread is empty. - * - * ⛔ WHAT IS DELIBERATELY NOT COPIED: ChatGPT's dark canvas and Glean's vendor logos. The palette - * is Loopable's own (`DESIGN.md`, and the font-token rule is gated app-wide over every CSS file - * in this tree), and the chips wear `FolderMark` — the SAME mark the nav rail draws for that - * database — because a brand logo for someone else's product is what those references have and - * this product's databases are not brands. "Exactly like" is about the interaction shape. - * - * ⚠ AND THE TOOL BOUNDARY IS NARROWER THAN THE SENTENCE "the chatbot can basically call the tools - * in our App". Under ruling R1 the assistant operates the caller's permitted database/view tools - * and nothing else: server-side there is exactly ONE tool, `build_view`. It cannot run an - * automation, touch a connector, or change a setting, and no part of this page implies it can. - */ -export default function AssistantPage({ granted }: AssistantPageProps) { - const [index, setIndex] = useState(initialIndex); - const [activeThread, setActiveThread] = useState(""); - const [selected, setSelected] = useState([]); - const [target, setTarget] = useState(""); - const [model, setModel] = useState("auto"); - const [question, setQuestion] = useState(""); - const [busy, setBusy] = useState(false); - const [problem, setProblem] = useState(""); - const [loaded, setLoaded] = useState(false); - const [confirmThread, setConfirmThread] = useState(""); - const [mode, setMode] = useState(readMode); - const [activeQuery, setActiveQuery] = useState(""); - /** Bumped on delete so the hosted workspace's own copy of the list cannot go stale behind us. */ - const [queryRefresh, setQueryRefresh] = useState(0); - const openCancel = useRef<(() => void) | null>(null); - const foot = useRef(null); - /** A counter, not a random id: the optimistic message only has to be unique within this page. */ - const pending = useRef(0); - - const databases = useMemo(() => databaseEntries(granted).filter((entry) => entry.kind !== "group"), [granted]); - const labels = useMemo(() => new Map(databases.map((entry) => [entry.key, entry.label])), [databases]); - /** - * ⭐⭐ WHICH CHIPS CAN ACTUALLY BE ASKED, and why the ones that cannot say so. - * The nav grants twelve databases on tenant #0 and the read boundary can answer for two: the - * ten `ut_odoo_*` grids are served through the connector mirror, which the Assistant refuses by - * design. Offering ten doors that cannot open — and letting the reader discover it one spent - * prompt at a time — is the same defect shape as a rail that paints two rows differently from - * five. ⚠ Absent from the map ⇒ TREATED AS ANSWERABLE: the server is the authority and its - * refusal is still the wall; a client that greyed out everything it had not heard about would - * hide a working source the moment this field failed to arrive. - */ - const sourceStatus = useMemo( - () => new Map(index.sources.map((row) => [row.database, row])), [index.sources]); - const blockedReason = useCallback( - (key: string) => (sourceStatus.get(key)?.answerable === false - ? sourceStatus.get(key)?.reason || "this database cannot be asked yet" : ""), - [sourceStatus]); - const messages = index.messages.filter((row) => row.threadId === activeThread); - const viewById = useMemo(() => new Map(index.views.map((view) => [view.id, view])), [index.views]); - - /** Offered is not callable: a model whose key this deployment does not hold says so, and why. */ - const modelBlocked = useCallback((key: string) => { - const row = index.modelStatus.find((item) => item.model === key); - return row && row.available === false - ? row.reason || "this model is not available on this deployment" : ""; - }, [index.modelStatus]); - - const reload = useCallback(async () => { - const result = await fetchQueries(); - setLoaded(true); - if (result.ok) { - setIndex(result.value); - // ⚠ Falls back to Auto when the SELECTED model is offered but not callable, not only when it - // has left the list: a stored thread can restore a model whose key was removed since, and a - // disabled option the picker cannot clear is a control that refuses its own value. - const callable = (key: string) => result.value.models.includes(key) - && result.value.modelStatus.every((row) => row.model !== key || row.available); - setModel((current) => callable(current) ? current : "auto"); - } - }, []); - - useEffect(() => { void reload(); }, [reload]); - useEffect(() => () => openCancel.current?.(), []); - useEffect(() => { - const permitted = databases.map((entry) => entry.key); - setSelected((current) => current.filter((key) => permitted.includes(key))); - setTarget((current) => permitted.includes(current) ? current : ""); - }, [databases]); - // The conversation reads bottom-up, like every chat the owner named. - useEffect(() => { foot.current?.scrollIntoView({ block: "end" }); }, [messages.length, busy]); - - /** - * The `?mode=` the redirect carried has been read into state; drop it so a toggle made later is - * not overridden by a stale parameter on the next reload. `replaceState` does NOT fire - * `hashchange`, so the shell never re-resolves the route and nothing flickers. - * - * ⛔ AND IT IS STORED IN THE SAME BREATH, WHICH IS THE HALF THAT WAS MISSING. `useState(readMode)` - * reads the hash and writes NOTHING; only `pickMode` persists. So a reader who arrived in Query - * by following a `#/query` link, never touching the toggle, would have the parameter stripped - * here and then land back in Chat on the next reload, restored from a choice made days earlier. - * Following the link IS choosing. - * - * ⛔⛔ AND IT LISTENS FOR `hashchange`, WHICH IS THE DEFECT THE OWNER REPORTED (QA 2026-08-17, - * measured on the deployed build). `#/query` and `#/assistant` resolve to the SAME route, so - * following a `#/query` link from anywhere inside the running app is a same-document hash - * change: this component is already mounted, `useState(readMode)` does not run a second time, - * and the panel stayed on Chat while the address bar read `?mode=query`. Only a full reload - * ever showed Query. **A mount-time read cannot see an arrival that does not remount** — the - * listener is the whole fix, and `arrivalMode` is the ONE parser both paths call so a later - * change cannot make the two disagree. - */ - useEffect(() => { - if (typeof window === "undefined") return; - const consume = () => { - const asked = arrivalMode(window.location.hash); - if (!asked) return; - setMode(asked); - storeMode(asked); - window.history.replaceState(null, "", `#/${ASSISTANT_ROUTE}`); - }; - consume(); - window.addEventListener("hashchange", consume); - return () => window.removeEventListener("hashchange", consume); - }, []); - - const pickMode = useCallback((next: AssistantMode) => { - setMode(next); storeMode(next); setProblem(""); - }, []); - - /** The saved Query views, grouped exactly as the standalone page groups them (ONE function). */ - const queryList = useMemo(() => queryGroups(index.views, databases), [index.views, databases]); - - const selectQuery = useCallback((qid: string) => { - setActiveQuery(qid); - openCancel.current?.(); - openCancel.current = openBuiltView(qid, true); - }, []); - - /** A renamed or duplicated artefact, merged in place, and the hosted workspace told to re-read. */ - const upsertQuery = useCallback((view: SavedQuery) => { - setIndex((current) => ({ - ...current, views: [view, ...current.views.filter((row) => row.id !== view.id)], - })); - setQueryRefresh((count) => count + 1); - }, []); - - const removeQuery = useCallback(async (qid: string) => { - const result = await deleteQuery(qid); - if (!result.ok) { setProblem(result.message); return; } - setIndex((current) => ({ ...current, views: current.views.filter((row) => row.id !== qid) })); - setActiveQuery((current) => (current === qid ? "" : current)); - setQueryRefresh((count) => count + 1); - }, []); - - /** Arriving in Query with nothing chosen selects the newest view of the first database. */ - useEffect(() => { - if (mode !== "query" || activeQuery) return; - const first = queryList[0]?.views[0]; - if (first) selectQuery(first.id); - }, [mode, activeQuery, queryList, selectQuery]); - - const newChat = useCallback(() => { - setActiveThread(""); setQuestion(""); setProblem(""); - }, []); - - /** Opening a past chat restores the sources and the model it ran with (R6). */ - const openThread = useCallback((id: string) => { - setActiveThread(id); setProblem(""); - const row = index.threads.find((thread) => thread.id === id); - if (!row) return; - const permitted = databases.map((entry) => entry.key); - const sources = (row.sources || []).filter((key) => permitted.includes(key)); - if (sources.length) { setSelected(sources); setTarget(sources[0]); } - if (row.model && index.models.includes(row.model) && !modelBlocked(row.model)) setModel(row.model); - }, [databases, index.models, index.threads, modelBlocked]); - - const toggleSource = useCallback((key: string) => { - if (blockedReason(key)) return; - setSelected((current) => { - const next = current.includes(key) ? current.filter((item) => item !== key) : [...current, key]; - setTarget((currentTarget) => next.includes(currentTarget) ? currentTarget : next[0] || ""); - return next; - }); - }, [blockedReason]); - - /** - * ⭐⭐ R16 — ENTER GOES TO THE CHAT, NOT TO A WAIT. Owner: *"Pressing enter goes immediately to - * the chat view showing that it is loading; there is a thinking animation."* - * - * This used to POST and only then touch `index`, so for the whole model call the reader sat on - * the opening screen looking at their own unsent sentence with nothing moving. The message is - * appended FIRST, which is what flips `empty` and hands the surface to the stream; `busy` then - * draws the app's one loading mark under it. - * - * ⛔ THE INVERSE IS CAPTURED BEFORE THE WRITE. An optimistic append that is not reconciled on - * failure leaves a message on screen the server never received, and the next turn replays it to - * the model as if it had [[undo-capture-before-the-write]]. The typed text goes back in the - * composer too: losing somebody's sentence to a failed request is the second injury. - */ - const ask = useCallback(async () => { - const text = question.trim(); - if (!text || !target || !selected.includes(target) || busy) return; - const pendingId = `pending_${(pending.current += 1)}`; - const optimistic: QueryMessage = { - id: pendingId, threadId: activeThread, role: "user", content: text, - createdAt: new Date().toISOString(), targetDatabase: target, - sources: selected, requestedModel: model, - }; - const withoutPending = (rows: QueryMessage[]) => rows.filter((row) => row.id !== pendingId); - setBusy(true); setProblem(""); setQuestion(""); - setIndex((current) => ({ ...current, messages: [...current.messages, optimistic] })); - const result = await submitChat({ question: text, database: target, sources: selected, - ...(activeThread ? { threadId: activeThread } : {}), model }); - setBusy(false); - if (!result.ok) { - setIndex((current) => ({ ...current, messages: withoutPending(current.messages) })); - setQuestion(text); - setProblem(result.message); - return; - } - setActiveThread(result.value.thread.id); - setIndex((current) => ({ - ...current, - threads: [result.value.thread, ...current.threads.filter((row) => row.id !== result.value.thread.id)], - // The server's own copy of the question replaces the optimistic one, so a thread never holds - // two of the same turn and the id the rest of the page keys on is the durable one. - messages: [...withoutPending(current.messages), result.value.userMessage, result.value.message], - views: result.value.view ? [result.value.view, ...current.views.filter((row) => row.id !== result.value.view?.id)] : current.views, - citations: [...current.citations, ...result.value.citations], - })); - }, [activeThread, busy, model, question, selected, target]); - - /** ⛔ The chat goes; the views it built STAY (they live in Query). Two-click, in the row. */ - const removeThread = useCallback(async (id: string) => { - const result = await deleteThread(id); - if (!result.ok) { setProblem(result.message); return; } - setIndex((current) => ({ - ...current, - threads: current.threads.filter((row) => row.id !== id), - messages: current.messages.filter((row) => row.threadId !== id), - })); - setActiveThread((current) => (current === id ? "" : current)); - setConfirmThread(""); - }, []); - - /** - * The rating goes to the server and the SERVER'S copy comes back, rather than the page patching - * its own guess: the reason is normalised and cleared there, and a rating that only looked - * applied is exactly the shape this module keeps being bitten by. - */ - const rate = useCallback(async (id: string, rating: "up" | "down" | null, reason?: string) => { - const result = await rateMessage(id, rating, reason); - if (!result.ok) { setProblem(result.message); return; } - setIndex((current) => ({ - ...current, - messages: current.messages.map((row) => (row.id === id ? result.value : row)), - })); - }, []); - - /** A built view opens WHERE THE READER ALREADY IS: the toggle flips, the list follows. */ - const preview = useCallback((qid: string) => { - pickMode("query"); - selectQuery(qid); - }, [pickMode, selectQuery]); - - const empty = messages.length === 0; - const composer = ( -
-
-